import pandas as pd
import pickle
import os
import numpy as np
from plotnine import *
import seaborn as sns
import matplotlib.pyplot as plt

import nltk
from textblob import TextBlob
from nltk.corpus import stopwords
from nltk.corpus import wordnet
from wordcloud import WordCloud
import re
from nltk.stem import WordNetLemmatizer
import heapq
from bs4 import BeautifulSoup
import lxml
import requests
import random
import collections
pd.set_option('display.max_rows',1000)
plt.style.use('ggplot')
%matplotlib inline

wd5 = '/Users/tmm/Documents/GitHub/STA160-Project/STA160-Jobs-Market-Project/Working/data'
os.chdir(wd5)

master_data_in = open('jobs_data.pickle', 'rb')
master_data = pickle.load(master_data_in)
master_data_in.close()

data_scientist_df_in = open('data_scientist_data.pickle', 'rb')
data_scientist_df = pickle.load(data_scientist_df_in)
data_scientist_df_in.close()

data_engineer_df_in = open('data_engineer_data.pickle', 'rb')
data_engineer_df = pickle.load(data_engineer_df_in)
data_engineer_df_in.close()

data_analyst_df_in = open('data_analyst_data.pickle', 'rb')
data_analyst_df = pickle.load(data_analyst_df_in)
data_analyst_df_in.close()

BI_df_in = open('BI_data.pickle', 'rb')
BI_df= pickle.load(BI_df_in)
BI_df_in.close()

master_data

1.) Data Exploration¶

Population Search Term Frequencies¶

How many listings are represented for each job type (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) within the population?¶

search_counts = master_data['Search'].value_counts()
search_counts

Data Engineer            349
Data Scientist            72
Data Analyst              49
Business Intelligence     49
Name: Search, dtype: int64

On the cybercoders website, given these four search terms the Data Engineer term is by far the most frequent listing. This may skew our data and will need to be accounted for in this analysis.

What is the percentage allocation for each job type (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) in our population?¶

search_freqDF = pd.DataFrame({'Frequency' : search_counts.values, 'Search' : search_counts.index, })
search_freqDF = search_freqDF[['Search','Frequency']]

percent1 = []
for i in range(len(search_freqDF)):
    holder1 = round(search_freqDF['Frequency'][i] / search_counts.values.sum(), 4)
    percent1.append(holder1)

search_freqDF['Percent of Total'] = percent1

search_freqDF

As noted previously, a large proportion of the listings scraped given the four search terms (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) are allocated to the Data Engineer position at 67.24%

(ggplot(master_data, aes(x = 'Search',fill = 'Search')) +
 geom_bar(stat = "count", position='stack') +
 labs(title = "Frequency Distribution for Data Scientist, Data Engineer, Data Analyst, and BI Roles", x = "Search", y = "Frequency")
)

<ggplot: (-9223372029288928263)>

plt.rcParams['figure.figsize'] = (10,20)
labels = search_counts.index[0], search_counts.index[1], search_counts.index[2], search_counts.index[3]
fracs = [search_counts[i] for i in range(len(search_counts))]
explode = [0.05, 0.05, 0.1, 0.05]
plt.axes(aspect=1)  
colors = ['lightcoral', 'cyan','lightskyblue','lightgreen']
plt.title('Pie Chart for Data Science Related Jobs (Data Scientist, Data Engineer, Data Analyst, Business Intelligence)')
plt.pie(x=fracs, labels=labels, explode=explode, autopct='%9.1f %%', colors=colors,
        shadow=True, labeldistance=1.1, startangle = 90, pctdistance = 0.8)
plt.show()

Population State Frequencies¶

What are the state frequencies representing our population?¶

state_counts = master_data['States'].value_counts()
state_counts

CA    255
NY     50
MA     27
VA     20
TX     15
WA     14
PA     12
MD     11
NC     11
IL      9
NJ      9
GA      8
MO      7
NV      6
MI      6
DC      6
TN      5
IN      5
CO      5
SC      5
AL      4
FL      4
MN      3
IA      2
RI      2
NH      2
NE      2
OH      2
ON      2
CT      2
OR      1
AB      1
ME      1
WI      1
KS      1
KY      1
UT      1
AZ      1
Name: States, dtype: int64

Here we discover that 255 of our listings are those located in the state of California. Intuition says that this may be due to the large number of technology jobs in the Silicon Valley/Bay Area. Given that we are interested in data science related jobs this makes sense.

state_freqDF = pd.DataFrame({'Frequency' : state_counts.values, 'States' : state_counts.index, })
state_freqDF = state_freqDF[['States','Frequency']]

percent2 = []
for i in range(len(state_freqDF)):
    holder2 = round(state_freqDF['Frequency'][i] / state_counts.values.sum(), 4)
    percent2.append(holder2)

state_freqDF['Percent of Total'] = percent2

state_freqDF

As noted previously, a large proportion of the listings scraped given the four search terms (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) are allocated to California at 49.13%. This covers almost half of the total states represented in our population.

objects = state_counts.index
y_pos = np.arange(len(objects))
performance = state_counts.values
 
plt.barh(y_pos, performance, align='center', alpha=0.5)
plt.yticks(y_pos, objects)
plt.xlabel('Frequency')
plt.title('Frequency Distribution of States Represented in Population')
 
plt.show()

Search Term Frequency Per State¶

How many listings are represented for each job type (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) within each state?¶

state_grouper = master_data.groupby('States')
jobs_per_state = state_grouper['Search'].value_counts()
jobs_per_state = jobs_per_state.reset_index(name = 'Frequency')
jobs_per_state

(ggplot(jobs_per_state, aes(x = "States", y = 'Frequency', fill="Search")) +
 geom_bar(stat = "identity",position='stack')+
 theme(figure_size=(15, 10),axis_text_x= element_text(rotation=80))+
 labs(title = "Distribution of Frequency of Jobs For Each State", x = "States", y = "Frequency")
)

<ggplot: (7565324566)>

As you can see from the above plot CA and NY stand out in that they represent each of the four job types. Next, we'll standardize the counts for each state given the search term to remove the skew due to the largest proportion of listings located in the state of California.

What are the proportions for each job type (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence) in each state?¶

Do some states have a greater proportion of their available listings allocated to a particular job type?¶

search_state_set = np.unique(jobs_per_state['States'])
search_state_dict = dict(tuple(jobs_per_state.groupby('States')))

search_freqDF = []
for state in search_state_set:
    search_state_freqDF = pd.DataFrame({'Frequency' : list(search_state_dict[state]['Frequency']), 
                                        'Search' : list(search_state_dict[state]['Search']),
                                        'States' : list(search_state_dict[state]['States'])})
    search_state_freqDF = search_state_freqDF[['States', 'Search', 'Frequency']]

    percent4 = []

    for i in range(len(search_state_freqDF)):
        holder4 = round(search_state_freqDF['Frequency'][i] / sum(list(search_state_freqDF['Frequency'])),4)
        percent4.append(holder4)
    search_state_freqDF['Percent of Total'] = percent4
    search_freqDF.append(search_state_freqDF)

search_freqDF = pd.concat(search_freqDF, ignore_index = True)
search_freqDF

(ggplot(search_freqDF, aes(x = "States", y = 'Percent of Total', fill="Search")) +
 geom_bar(stat = "identity",position='stack')+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Job Proportions For Each State", x = "States", y = "Allocation")
)

<ggplot: (-9223372029288966249)>

After converting the frequency of listings for each job type given a state we are able to have a better look at the variety of data science related options for each state. Some noteworthy states are as follows:

Four job types represented: CA, NY

Three job types represented: GA, IN, MA, MO, NC, NJ, PA, TX, VA, and WA

Later on we'll analyze skills and if you are interested in more than one of the given job types the above states may be a good place to look for a variety of data science related positions.

City Frequencies Per State¶

What are the frequencies for each city within the represented states?¶

Do any cities in particular stand out which may indicative of a hotspot for data science jobs?¶

city_grouper = master_data.groupby('States')
jobs_per_city = city_grouper['Cities'].value_counts()
jobs_per_city = jobs_per_city.reset_index(name = 'Frequency')
jobs_per_city = jobs_per_city.sort_values(by = 'Frequency',ascending = False).reset_index(drop = True)
jobs_per_city

For now we'll ignore the fact that California is vastly represented in our population and can conclude that the following cities may be great places to search for Data Scientist, Data Engineer, Data Analyst, and Business Intelligence jobs.

1.) San Francisco, CA 2.) New York City, NY 3.) San Mateo, CA 4.) San Jose, CA 5.) Irvine, CA 6.) Los Angeles, CA 7.) Palo Alto, CA 8.) Seattle, WA 9.) Redwood City, CA 10.) Cambridge, MA

The idea behind why we feel that it is reasonable to ignore the much larger proportion of listings located in California is based on the fact that many of the technology companies which are known to have more data science positions are located in the Bay Area. We can see from above that five out of the top 10 represented cities are located in this area.

(ggplot(jobs_per_city[:20], aes(x = "States", y = 'Frequency', fill="Cities")) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 5),axis_text_x= element_text(rotation=80))+
 labs(title = "Distribution of Frequency of Cities For Each State", x = "State", y = "City Frequency")
)

<ggplot: (-9223372029310727038)>

In order to support our previous statement, California stands out because compared to the other states there are many different options for city locations within that state. Unlike the other top city states where there are no more than two city options.

How does the number of cities proportionate to the total number of listings for a particular state effect the distribution of listings in our population?¶

search_cities_set = np.unique(jobs_per_city['States'])
search_cities_dict = dict(tuple(jobs_per_city.groupby('States')))

search_cit_freqDF = []
for city in search_cities_set:
    search_cities_freqDF = pd.DataFrame({'Frequency' : list(search_cities_dict[city]['Frequency']), 
                                         'Cities' : list(search_cities_dict[city]['Cities']),
                                         'States' : list(search_cities_dict[city]['States'])})
    
    search_cities_freqDF = search_cities_freqDF[['States','Cities','Frequency']]

    percent5 = []

    for i in range(len(search_cities_freqDF)):
        holder5 = round(search_cities_freqDF['Frequency'][i] / sum(list(search_cities_freqDF['Frequency'])),4)
        percent5.append(holder5)
    search_cities_freqDF['Percent of Total'] = percent5
    search_cit_freqDF.append(search_cities_freqDF)

search_cit_freqDF = pd.concat(search_cit_freqDF, ignore_index = True)
search_cit_freqDF = search_cit_freqDF.sort_values(by = 'States',ascending = True).reset_index(drop = True)
search_cit_freqDF

(ggplot(search_cit_freqDF, aes(x = "States", y = 'Percent of Total', fill="Cities")) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 5),axis_text_x= element_text(rotation=80))+
 labs(title = "City Proportions for Each State", x = "State", y = "City Allocation")
)

<ggplot: (7566853117)>

You will notice above that states with a greater number of cities represented given our listings is indicative of a wider range of options for relocation when thinking about your job search.

California, Colorado, Massachussetts, Maryland, and North Carolina have a large number of cities represented by our listings meaning a job seeker has options for the same job title but in different cities with a preferred state.

Data Scientist Listing Frequencies Per City¶

What are the top 10 cities in the USA for a person seeking a Data Scientist position?¶

ds_city_grouper = data_scientist_df.groupby('Location')
ds_per_city = ds_city_grouper['Search'].value_counts()
ds_per_city = ds_per_city.reset_index(name = 'Frequency')
ds_per_city = ds_per_city.sort_values(by = 'Frequency',ascending = False).reset_index(drop = True)
ds_per_city

TOP_ds = ds_per_city[:10]

(ggplot(TOP_ds, aes(x = "Location", y = 'Frequency', fill='Location')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Data Scientist - Distribution of Frequency of Jobs For Top 10 Most Frequent Locations", x = "Location", y = "Frequency")
)

<ggplot: (7543421732)>

If you are seeking a data scientist position, your best option is to look in New York City - NY and San Francisco - CA.

Data Engineer Listing Frequencies Per City¶

What are the top 10 cities in the USA for a person seeking a Data Engineer position?¶

de_city_grouper = data_engineer_df.groupby('Location')
de_per_city = de_city_grouper['Search'].value_counts()
de_per_city = de_per_city.reset_index(name = 'Frequency')
de_per_city = de_per_city.sort_values(by = 'Frequency',ascending = False).reset_index(drop = True)
de_per_city

TOP_de = de_per_city[:10]

(ggplot(TOP_de, aes(x = "Location", y = 'Frequency', fill='Location')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Data Engineer - Distribution of Frequency of Jobs For Top 10 Most Locations", x = "Location", y = "Frequency")
)

<ggplot: (7565478734)>

If you are seeking a data engineering position, your best option is to look in San Francisco - CA, San Jose - CA, San Mateo - CA, and New York City - NY.

Data Analyst Listing Frequencies Per City¶

What are the top 10 cities in the USA for a person seeking a Data Analyst position?¶

da_city_grouper = data_analyst_df.groupby('Location')
da_per_city = da_city_grouper['Search'].value_counts()
da_per_city = da_per_city.reset_index(name = 'Frequency')
da_per_city = da_per_city.sort_values(by = 'Frequency',ascending = False).reset_index(drop = True)
da_per_city

TOP_da = da_per_city[:10]

(ggplot(TOP_da, aes(x = "Location", y = 'Frequency', fill='Location')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Data Analyst - Distribution of Frequency of Jobs For Top 10 Most Locations", x = "Location", y = "Frequency")
)

<ggplot: (7565324720)>

If you are seeking a data analyst position, your best option is to look in Las Vegas - NV, New York City - NY, Atlanta - GA, Princeton - NJ, and Ventura - CA.

Business Intelligence Listing Frequency Per City¶

What are the top 10 cities in the USA for a person seeking a Business Intelligence position?¶

bi_city_grouper = BI_df.groupby('Location')
bi_per_city = bi_city_grouper['Search'].value_counts()
bi_per_city = bi_per_city.reset_index(name = 'Frequency')
bi_per_city = bi_per_city.sort_values(by = 'Frequency',ascending = False).reset_index(drop = True)
bi_per_city

TOP_bi = bi_per_city[:10]

(ggplot(TOP_bi, aes(x = "Location", y = 'Frequency', fill='Location')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Business Intelligence - Distribution of Frequency of Jobs For Top 10 Most Locations", x = "Location", y = "Frequency")
)

<ggplot: (7543659496)>

If you are seeking a business intelligence position, your best option is to look in Irvine - CA and White Plains - NY.

2.) Analysis - Preferred Qualifications¶

stop_words = stopwords.words("english") + ['data',
                                           'analysis',
                                           'development',
                                           'design',
                                           'modeling',
                                           'systems',
                                           'analytics',
                                           'test',
                                           'management',
                                           'security',
                                           'server',
                                           'system',
                                           'engineering',
                                           'model',
                                           'experience', 
                                           'data', 
                                           'structures',
                                           'science',
                                           'software',
                                           'algorithms',
                                           'testing',
                                           'business']

skills_df = master_data.copy()

for i in range(len(skills_df)):
    skills_df.loc[i, 'Skills'] = skills_df['Skills'][i].replace("Machine Learning",'Machine-Learning')
    skills_df.loc[i, 'Skills'] = skills_df['Skills'][i].replace("Big Data",'Big-Data')
    skills_df.loc[i, 'Skills'] = skills_df['Skills'][i].replace("Tekla Structures",'Tekla-Structures')

skills_set = set(skills_df['Skills'])

skills_set

{'.NET, C#, SQL, Agile Software Development, ASP.NET, NoSQL, AngularJS',
 '.NET, C#, SQL, Team Foundation Server (TFS), Agile Software Development',
 '.NET, C#, Windows, Big-Data, SQL Server',
 '.NET, Visual Studio, C#, SQL, MVC',
 'AI, Marketing, Strategic Partnerships, Business Development, Artificial Intelligence, Machine-Learning',
 'API Development, Large Data Sets, C#, Java, Python, AWS, REACT, Node.js, Agile Methodologies, elastic search',
 'ASIC/SoC, Verification Flows, Verification process, SV/UVM environments, SoC architecture/design, CPU, Data center protocols, Ethernet, Startup Experience',
 "ASP.NET MVC, C#, JavaScript, WCF, RESTful API's",
 'ASP.NET MVC, C#/.NET, SQL Server, Developing Custom Internal Business Applications',
 'ASP.NET, MVC, C#, JQuery, SQL, Entity Framework',
 'AWS, Big-Data, Python, Production Level Coding',
 'Ability to be client facing, Mathematics, Can interpret data, Excel',
 'Account Executive, Sales, SaaS, Business Intelligence, B2B Sales',
 'Air Quality Assessment, Environmental Engineering, Emissions Reporting, Emissions Inventories, Data Analysis, Title V, Reclaim, Project Management, Database Management, MS Office',
 'Algorithm Development, Machine-Learning, Computer Vision, Image Processing, 3D Modeling, Object Detection, 3D CT Data, Prototyping, Signal Processing, Python C++',
 'Algorithm Development, Machine-Learning, Image Processing, 3D Modeling, Object Detection, 3D CT Data, Prototyping, Signal Processing, Matlab, Python C++',
 'Algorithm Development, Machine-Learning, Image Processing, matLab Pyton C/C++, 3D Modeling, Image Processing and analysis, Object Detection, 3D CT Data, Prototyping, Signal Processing',
 'Analytics, Data Analysis, Business Intelligence, SQL',
 'Android, Data Structures, Algorithm Design, Java',
 'Angular, Microsoft WebAPI/C#, OSIsoft, AF SDK, AF Templates, Data References, Analysis, Agile, TFS, TFVC',
 'Angular, TypeScript, Cordova, Agile, Kanban, Test Driven Development, Continuous Delivery, .NET, Data Science',
 'Angular, TypeScript, JavaScript, HTML, CSS',
 'Application Support, Business Analysis, Systems Analysis, SQL, Healthcare Industry, .NET, XML, Technical Support, Data Analysis, Client Interaction',
 'Applied Research, Hadoop/MapReduce/Spark, Programming (R/Scala/Python/Ruby), Java Or C++, Data Mining, Machine-Learning, Information Retrieval',
 'ArcSight, Network/Host/Data/Application Security, IP Networking, Networking Protocols, security related technologies, Internet/Web/Application/Network Security, Windows/Solaris/Linux, Firewall/Network Scanning/Intrusion Detection, Federal Regulations/Information Security, NIST Special Publications/C&A Process Methodology',
 'Azure SQL, PowerBI, Warehouse design',
 'BI Reporting, Data Integration, ETL, data documentation deliverables, Analytics, Data Delivery, SQL, Data Warehousing',
 'Big-Data, ADTECH, Hadoop, Scala, SPARK',
 'Big-Data, AWS, MongoDB, SQL',
 'Big-Data, Architecture, Hadoop',
 'Big-Data, Big O Notation, Scala, Hadoop, Data Mining, Kafka, GIT, Unix, Akka Streams, Maven',
 'Big-Data, Data Ingestion, Data Transformation',
 'Big-Data, Machine-Learning, Artificial Intelligence, Scala, Java, Python, AWS',
 'Big-Data, Python, AWS',
 'Big-Data, Scala, Hadoop, Data Mining, Kafka, GIT, Unix, Akka Streams, Maven',
 'Big-Data, TS/SCI, CI polygraph, Hadoop',
 'Big-Data/HPC Programming, Java, C/C++/C++11, SQL, Scala, Machine/deep learning algorithms, HDFS/Spark/Cassandra/TensorFlow, Product Engineering, MySQL, Linux Kernel',
 'Black Box, Automated Testing, Sanity Testing, Boundary, Negative, Compatibility, Concurrency, Test Data Sets, quantitative reports',
 'Blockchain, Cryptography, AI, Java/Python/Golang/JS/Ruby, Tensor Flow, Data Science',
 'Blockchain, Ethereum, Python, C++, Blockchain test tools (Truffle), Blockchain management tools, Continuous Integration, DevOps environment, Node.js, JavaScript',
 'Business Analysis, 4 year degree, Experience rel. to Internal Policies & Procedures, training experience',
 'Business Analyst, Business Analysis, SQL, Data Analysis, Agile, Excel',
 'Business Analytics, Python, SQL, Data Analysis, Excel, Site Catalyst',
 'Business Intelligence, Data Warehousing, AWS, Azure, Analytics, Amazon RedShift, Oracle, Business Objects, Tableau, PowerBI',
 'Business Intelligence, MS SQL Server, Ssrs, Ssis, ETL, Business Analysis, Dashboards, Reports, Tableau',
 'Business Intelligence, Relational Databases, SQL Queries, Power BI, Ssrs, T-SQL',
 'Business Process Standardization, 4 year degree, Training',
 'C#, .NET, CLR, Windows',
 'C#, .NET, Microsoft Visual Studio, Object-oriented Programming, WPF, MVVM, OPC - OPC Data Manager, UML, Multi-Threading',
 'C#, ASP.NET, JavaScript, T-SQL, JQuery, Data Modeling, Unity, Docker, RESTful, IIS',
 'C#, SQL, ASP.NET, ERP System Development, Visual Studio, JavaScript, Data Modeling, Manufacturing experience, Logistics Experience',
 'C#/.NET, SQL Server, Some CRM Development/Customization',
 'C++, Data Storage, Network File System, Storage Programming, REST Architecture, Network Programming',
 'C++, EDA, STL, Power Analysis Tools, Signal Integrity, extraction, OOD, EDA Algorithms, Data Structures, Phyiscal Verification',
 'C++, Linux - RedHat, Embedded Software Development, Commercial Products Development, Multithreaded and Object Oriented Design, DNP3 a plus, MySQL - Oracle - or other database, Networking experience with TCP/IP, Data analytics in a large big-data framework, Hadoop clusters is a plus',
 'C++, Linux Platform, Search Algorithms, Information Retrieval, Data Analytics, C++ 11, Python Scripting',
 'C++, Machine-Learning, Data Mining',
 'C++, Medical Devices, Startup Experience, Data Visualization, state machine',
 'C++, Python, JavaScript, TDD / BDD, Large-Scale big data applications, Financial Services / Capital Markets, Unix Command Line, Data Structures, Full-Stack Development',
 'C++, search engine technologies, Search Algorithms, Information Retrieval, Data Analytics, C++ 11, Linux Platform, Python Scripting, Distributed Algorithms - GPU and Deep Learning, Distributed Systems',
 'C, C++, Java, TS/SCI CI Poly, Python, RUBY, Perl, Big-Data analytics, Amazon Web Services, TCP/IP',
 'CAD development, C/C++, Windows, Parasolid, advanced data structures',
 'CCIE, Cisco R/S, Cisco Data Center, Cisco Security, Cisco ISE, Sourcefire, Pre Sales',
 'CCIE, Pre Sales, Cisco Data Center, Enterprise Networking, Cisco UC / Collaboration, Cisco Security',
 'Cisco R/S, Cisco Data Center Solutions, UCS, Nexus, CCIE, CCNP, VCP, Post Sales',
 'Cisco R/S, Cisco Security, Cisco Voice, Cisco Data Center, Pre Sales, Cisco UCS, CCNP, CCDP',
 'Computer Vision, Hands-on leadership, Python, Tensorflow, R, Machine-Learning, Deep Learning, OpenCV/PCL, Revit, 2d/3D Spatiotemporal Data',
 'Core Java, Big-Data, Web programming, cloud platforms, Relational Databases, CI Tools',
 'Cost Management, Large Capital Projects, Project control fundamentals, Financial/schedule analysis, Budgeting methodologies, Advanced data manipulation in Excel, Change Management OR Project Management',
 'Cost Management, Planning, Scheduling, Large Capital Projects, Project control fundamentals, Financial/schedule analysis, Budgeting methodologies, Advanced data manipulation in Excel, Primavera OR MS Project',
 'Create test plans/cases, QA in an e-Commerce environment, Perform end to end testing, Write basic SQL queries for data driven tests, Selenium WebDriver Automation, Manuel testing to find bugs and report, Work with automation to learn automation framework, Review results from automated testing team, Use test tracking software, Write QA automation tests',
 'Credit Risk Modeling/Response models, Multivariate Analysis, Machine-Learning, R, Python, Data Mining, Hadoop',
 'Customer Success Experience, Customer Renewals, Business Intelligence, Enterprise Software, SaaS',
 'Data Analysis, Business Intelligence, E-Commerce, Data Management, ETL processes',
 'Data Analysis, Data visualization/Dashboards, Tableau, Data Standardization/Cleansing, People Management, R programming, SQL, ETL, KPI Reporting/Analysis, Excel',
 'Data Analysis, Data visualization/Dashboards, Tableau, Data Standardization/Cleansing, R programming, SQL, ETL, KPI Reporting/Analysis, Excel, People Management',
 'Data Analysis, Lifestyle Polygraph, Systems Engineering, Intelligence, Full Scope',
 'Data Analysis, SQL, E-Commerce, Retail analytics, Google Analytics, Tableau, SalesForce',
 'Data Analytics Platforms, Machine-Learning, Machine vision, Python, Java, C++',
 'Data Center Solutions, NetApp, VMWare, Cisco, Virtualization, Post Sales, Implementations',
 'Data Center Solutions, NetApp, VMWare, NCDA, NCIE, Cisco, Virtualization, Post Sales',
 'Data Engineering, Data Visualization, Hadoop, Kinesis, Kafka, SPARK, Flink, Java, RUBY, Python',
 'Data Interpretation, Writing Algorithms, LabView, Python, C++, Matlab, RF Test, Microwave, Communication Interfaces, Antenna',
 'Data Mining, SAS, Tableau, R, Python, SQL, Data Visualization',
 'Data Mining, data sets, Statistical Modeling, Python & R & SQL, Machine-Learning Algorithms, Web Services, Hadoop',
 'Data Pipeline, Python, Machine-Learning',
 'Data Pipelines, Datat Structures, AWS/Cloud Infrastructure, Platform Infrastructure, Big-Data-Hadoop/kafka/storm/spark/zookeeper/hbase',
 'Data Pipelines, Java, Kafka',
 'Data Science Experience, numpy, Machine-Learning',
 'Data Science or Engineering, Python, R, Machine-Learning, Data Mining, Hadoop, SQL, Java, SPARK, Scala',
 'Data Science, Applied Research, Hadoop/MapReduce/Spark, Programming (R/Scala/Python/Ruby), Java Or C++, Data Mining, Machine-Learning, Information Retrieval',
 'Data Science, Artificial Intelligence, Statistical Analysis, Data Modeling, Machine-Learning, Algorithms',
 'Data Science, Big-Data, Machine-Learning, Artificial Intelligence, Scala, Java, Python',
 'Data Science, Machine-Learning, Internet of things, Smart Manufacturing, Industrial Internet of Things, SQL Databases, Cloud Computing, Programming (C/C++ - C# - Python - Java), Manufacturing',
 'Data Science, Machine-Learning, Natural Language Processing, Big-Data, Python',
 'Data Science, Machine-Learning, Natural Language Processing, Python, Text Analysis, Recommendation Systems, Hadoop, SPARK, Distributed Computing, Information Retrieval',
 'Data Science, Machine-Learning, Python, scalability of code & product & modeling, closed-looped analytics, AI product expertise',
 'Data Science, Machine-Learning, R, Python, SQL, SQL Server, E-Commerce',
 'Data Science, Machine-Learning, Simulation, Python, numpy',
 'Data Science, Python, scikit.learn, Bayesian Inference, Machine-Learning Algorithms',
 'Data Science, Python, scikit.learn, Virtual Environments, Bayesian Inference, Machine-Learning Algorithms, Prior teaching experience (not essential)',
 'Data Science, Python, scikit.learn, Virtual Environments, Bayesian Inference, Machine-Learning Algorithms, teaching experience',
 'Data Security Administration, CSSP, CEH, GCIH, SCYBER, GCIA, GCFA, DCO, AFSCN, MILSATCOM',
 'Data Storage, Ruby On Rails, REACT, Python, JavaScript, API Development',
 'Data Structures, Algorithms, Systems Behavior, log files, Infrastructure, DTrace/ Btrace/AspectJ, Splunk/ELK, OpenTSDB',
 'Data Structures, Machine-Learning Algorithms, Cloud Computing, Scripting languages, Python, ATHENA, MySQL, Hadoop, Looker, Build Systems',
 'Data Structures, Object-oriented Programming, JavaScript, REST/JSON, Design Patterns, UI/UX Collaboration',
 'Data Structures/Algorithms/Systems, Interconnected Systems, Configuration, Datacenter Infrastructure, Configuration Management Systems, Chef/Puppet/BMC Remedy/Ansible/Docker',
 'Data delivery and streaming systems (Kafka/Storm), Java, Scala, Data Pipelines',
 'Debian, Linux, Java, Python, MySQL, Shell Scripting, HTTP/S, SMTP, SSL, Nagios',
 'Deep Learning, Machine-Learning, Natural Language Processing, NLP, Tensorflow, PyTorch',
 'Delivery, Retail, E-Commerce, client-facing, Data Analysis, Business Intelligence, Data Management, ETL processes',
 'Develop high quality code using APEX, Java, Design/develop solutions on the salesforce.com, Visualforce and Apex, Marketing and sales cloud, Apex classes/triggers and Visualforce pages, SFDC Platform I & II certification, NodeJS, Lightning and Zuora, Salesforce data tools -Data Loader/Excel/Connector',
 'Developing Automated Integration/Unit Tests, Test Cases, Python, Linux, Systems Level, Automation Testing, Cloud environments, Big-Data Environments, Cloud Automation',
 'Developing Hybris eCommerce sites, Hybris Accelerators & core platform, APIs & tools-i.e. HAC/HMC & the Hybris Cockpits, Spring Core & MVC, Java with JEE, RDBMS such as Oracle or MySQL, Real time data exchange via REST or SOAP-based web, Asynchronous file exchange, Test Driven Development and JUnit, Hybris suite/stack and its APIs',
 'Devops, Continuous Automation, Continuous deployment, Linux System Administration, Amazon Web Services (AWS), Network Administration, Configuration management (Chef/Ansible/Puppet/etc), Jenkins, Bash, Python',
 'Docker, AWS, cl/cd, Jenkins, GIT, Big-Data',
 'ETL, AWS, Informatica, DB2, Postgres, Kimball, Linux, Data Warehousing, Agile, Data Modeling',
 'ETL, Business Intelligence, SQL Server',
 'ETL, RDBMS, Tableau, SQL, API, NoSQL, MongoDB, DTS, Business Intelligence',
 'ETL, SQL, Big-Data',
 'ETL, SQL, C#, .NET, BI, T-SQL, Ssis, Ssrs, GIT, TFS',
 'ETL, SQL, Database Development, Database Design, SDLC',
 'ElasticSearch, Cassandra, Hadoop, AWS, EC2, S3, Chef, Ansible, Python, RUBY',
 'Electrical Design, Data Center, Construction, Building Design, Mission Critical Design, AutoCAD, MS Project, Electrical Infrastructure, Professional Engineering License',
 'Embedded Product Development, Linux, M2M Technology, IOT, C/C++ Prorgramming, Network Protocols, Communication Systems, Algorithms and Data Structures, JavaScript/HTML/CSS/Bash',
 'Embedded, AI, IOT, AWS, Node.js, Network Security, Algorithms, Full Stack Development',
 'Excel, Oracle, SQL, Metadata, Reference Data Management, Master Data Management',
 'Excel, Power BI, Tableau, .NET, Java, Pre-Sales, AWS, C#, .NET, ASP.NET',
 'Extrusion, Polymer, Medical Device, Data Mining, Continuous Improvement, Equipment, Prototype, supervision, Expansion, new technology',
 'FPGA, RTL, UVM, data storage: Internal FPGA RAM, SRAM, DDR, DDR2-3',
 'Financial Analysis, Excel, Financial Modeling, Forecasting, Financial Analyst, Google Analytics, SQL, MS Query',
 'Financial Analysis, Excel, Financial Modeling, Forecasting, Financial Analyst, Google Analytics, SQL, MS Query, GAAP, Budgeting',
 'Financial Analysis, Financial Modeling, Budgeting / Forecasting, Business Intelligence Analysis, Budget vs. Actual Analysis, Analyze Cost, Financial Presentations, Financial Services Industry',
 'Financial Analysis, Manufacturing (high-volume durable goods), Oracle ERP (preferred), Data analytics and forecasting',
 'Financial Systems, General Ledger, User Security, Accounts Payable, Infor-11, Reporting, Test plans/scripts, SQL and Relational Database Concepts, C#/JavaScript, HRIS Payroll Data',
 'Frontend development, Vue/Angular/React, HTML5, CSS3, JavaScript, UX/UI, Data Visualization, Responsive Design, API Development, Mobile/Web',
 'Full Stack, Cassandra, ElasticSearch, Solr, Data Scientist, Python, Machine-Learning, R, Data Mining, SQL',
 'Go / Golang, Machine-Learning, Software Engineering',
 'Go, IOT, Big-Data, Video, Protobuf, Python, Multithreaded Applications, TCP/IP, Concurrency Management',
 'Go, Python, Java, C#, C++, Scala, Kotlin',
 'HMI, PLC, AutoCAD, Robotics, Machine Control Systems, Motion Control Systems, Data acquisition systems, Allen Bradley, Omron, Servo systems',
 'Hadoop, Hive, Mapreduce, yarn, Kafka, Pig, Oozie, hBase',
 'Hadoop, Hive, Spark and Presto, Apache Solr and/or Apache Lucene',
 'Hadoop, Java, Linux, Bash/Perl scripting, Apache, Kafka, SPARK, Mapreduce, AWS, Scala',
 'Hadoop, Mapreduce, Java, Python',
 'Hadoop, Mapreduce, SPARK, ElasticSearch, Java, Scala, Python',
 'Hadoop, Pipeline Infrastructure, Pig, Hive, SPARK, Mapreduce, Kafka, Kinesis, Storm, Vertica',
 'Hadoop, SQL, SPARK, Kafka',
 'Hadoop, Scala, SPARK, Java, NODE, REACT, Big-Data',
 'Hadoop/Spark/Hive, Quantitative Analysis, Python/R, SQL, Big-Data Systems (Hadoop/Hive/Spark/)',
 'Healthcare, Financial Analyst, Medical Billing, Data Analysis, Microsoft Access, SQL Scripting, Crystal Reports, Managed Care',
 'Healthcare/ Defense/Telecommunications Industry, Data Modeling, Data Warehousing, ETL, SPARK, SQL or NoSQL, Spark/ Scala/ Hadoop/ Storm',
 'ISO 13485, Supplier Audits, PPAP, Mechanical or Industrial Engineering degree, Process Validation, Process Capability Analysis, Root Cause Analysis, Statistical/Data Analysis Skills',
 'ISO 9001, ISO 13485, Quality Leadership / Management, QMS, Quality Assurance, Manufacturing, Six Sigma, Internal Auditing, Data Analysis',
 'Informatica, SQL, ETL Development, Teradata, Data Modeling, Data Warehousing',
 'Information Management, Data Analysis, Business Intelligence, SQL, Integration tools, Data Modeling, CASE tools, OLAP',
 'Information Security, CISSP or CISA or CCNP or CEH, HIPAA, ISO 900, NIST, HITRUST, SOX, NAC, Data Encryption, data loss prevention',
 'Insurance Industry, Data Science, Text Analytics, Statistical Modeling in Business settings, Ridge/Lasso/Elastic Nets, SAS/R/H2O/Python/Spark/SQL/Hadoop, Git/Gitlab, Data visualization: Shiny/Spotfire/Tableau, Tree models: Random Forest/GBM, linear/logistic regression',
 'Insurance, Claims, Bodily Injury, Litigation, Coverage, damages, Estimates',
 'J2EE, Java, SQL, JDBC, Microservices, RESTful, Hibernate',
 'JD Edwards Enterprise One 9.1, OneView, ASP.NET, C#, bi publisher, CafeOne, PowerForms, transactional SQL proces',
 'JD Edwards Enterprise One, Sales and Distribution, One View, bi publisher, SQL, CNC, EDI',
 'Java, AWS, ETL, GIS, GIT, data-driven development, Rest, Go, Linux',
 'Java, AWS, SPARK, Python, Hadoop, EMR, Scala',
 'Java, C++, C#, Data Structures, Algorithms',
 'Java, C++, Python, JSON, MongoDB, MySQL, Cassandra',
 'Java, Customer Facing, Professional Services, Client Facing Development, Distributed Systems, Cloud/AWS/EC2/EMR/Redshift/Azure, SQL',
 'Java, Data Structures, Algorithm Development, System Design/Development, Linux Or Unix, OOD, Agile Program Management',
 'Java, Data persistence layers, Video mobile and social network applications, Android Development, Android SDK, Published Android Applications, Android Application Architecture, Android libraries, Android automated testing frameworks, Clean coding standards',
 'Java, Go, C++, Zookeeper, Dropwizard, Kafka, Cassandra',
 'Java, Go, Zookeeper, Dropwizard, Kafka, Cassandra, RUBY',
 'Java, Golang, Go, AWS, Big-Data',
 'Java, Hadoop, Cloudera, MarkLogic',
 'Java, Hadoop, Python, NoSQL, AWS, Machine-Learning',
 'Java, J2EE, Apache Storm, Kafka, ElasticSearch, kibana, hBase, Graph DBs, Data Modeling, Business Intelligence',
 'Java, J2EE, Machine-Learning, Big-Data, R, Python, LAMP, Database Infrastructure, Data Architecture',
 'Java, J2EE/Spring/Hibernate, Multi-Threaded Development, Data Mining, Prolog',
 "Java, J2EE/Spring/Hibernate, SQL, RESTful API's, data mining software",
 'Java, JEE, JBoss, WildFly, Data Structures, Algorithms, REST API, JAX, Unix/Linux, PostgreSQL',
 'Java, JUnit, JMeter, Big-Data Experience (Hadoop/MongoDB/Cassandra), Cloud Data Technologies- AWS, SQL, Distributed Systems, Large Scale Distributed Application Development, SOLR or Lucene or ElasticSearch',
 'Java, JavaScript, Python, Expert HTML5 and CSS3, SQL, API',
 'Java, JavaScript, Spring, Angular/React, Big-Data',
 'Java, Node.js, Python, RESTful APIs, Big-Data, Data Mining, Cloud Computing, JavaScript',
 'Java, Object Oriented, Data Structures, Algorithm Development, System Design/Development, Leading code reviews',
 'Java, Python, Go, Test Frameworks (home grown), OSS frameworks',
 'Java, Python, SQL Relational Database, Hadoop / Pig / Hive / HBase, Scala',
 'Java, SQL, JavaScript, AWS, Hadoop/Big-Data',
 'Java, SQL, React.JS, JavaScript, AWS',
 'Java, SaaS, JavaScript, HTML/CSS, Spring, Boot, SQL Server, Server-Side Testing',
 'Java, Scala, NoSQL, Rest',
 'Java, Scala, SPARK, Hadoop, Distributed Systems, Python',
 'Java, Software Development, Spring, Hibernate, Big-Data',
 'Java, Spring Boot, Hibernate, AWS, Cloud, Unit Testing',
 'Java, Spring, Big-Data',
 'Java, Spring, SOA, Bootstrap',
 'Java, Spring, Sacalability, High Availability, Multithreading, Enterprise Class Environments, Data Processing, Big-Data, Cloud Computing (AWS), Continuous Delivery',
 'Java, Spring, TDD, SPARK, Data conversions, Avro, Parquet, Orc, Hadoop',
 'Java, UI Development, YUI, / JSF / Java Server Faces GWT / DOJO, Spring / Hibernate, Pulling / Retrieving / Exposing Data & APIs, SQL scripts, Computer Science Fundamentals, Object Oriented Programming best practices, Algorithms / Data Structures',
 'Java, Web Services, SOAP, Rest, Data Analysis, Data Modeling, design concepts',
 'Java/J2EE Development, Linux/UNIX, Multi-tier web applications, Web Services/SOA/XSD',
 'JavaScript, Angular, REACT, Script and Styling Language Expertise, Web Services Expertise, Front-end E-commerce Experience, Linux, MySQL, Data Analytics Expertise, Unit/Integration Testing Expertise',
 'JavaScript, ES6 syntax, Responsive HTML5 web-applications, UI/UX, MVC Framework, CSS, Database management and SQL, Node.js, SAP HANA platform and data modeler, SAPUI5 or openUI5 library',
 'JavaScript, PHP, Laravel, AWS, Angular/React/Marionette, Cassandra/MongoDB',
 'JavaScript, REACT, CSS3, HTML5, Hadoop, Scala, SPARK, Java, NODE, Big-Data',
 'JavaScript, React.JS, Golang, Open Source, Data Visualization, TDD, API, GIT, D3, Jasmine',
 'JavaScript, SaaS, Bootstrap, React or Angular, Data Visualization, HTML5 / CSS, ASP.NET, SQL Server',
 'JavaScript, TypeScript, Node.js, Python, Object-oriented Programming, Functional paradigms, data structure, Algorithm, Multi-Threading, Socket',
 'JavaScript, User Interface Design / Development, SaaS, Responsive Web Development, Bootstrap, Visual Studio, HTML5, CSS3, d3.js, JIRA',
 'JavaScript, XHTML, CSS, AJAX, Agile Development Methodologies',
 'JavaScript/HTML5/CSS3, Java, AngularJS, T-SQL, J2EE, Big-Data',
 'Kafka, Data, Kafka implementation, Confluent Kafka, Java, SPARK, Scala, Hadoop',
 'Kimball, iSeries, SQL, T-SQL, SISS, Ssrs, Tableau, Datazen, MySQL, Oracle',
 'LIMS, Java/Spring, Network Data Transfers, Microservices, Linux, Lab Info Management System',
 'LabView, TestStand, Zigbee, WiFi, cellular, Ethernet protocols, USB, C/C++, Python',
 'LabView, Visual Basic, C++, C#, SQL, Comm Interfaces (USB/GPIB/Ethernet/RS232), Simulation, System Integration, Motion Control, data acquisition',
 'Large Data Sets, statistical and predictive modeling, Machine-Learning',
 'Linux, AWS, Java',
 'MS SQL, T-SQL, Ssrs, .NET Framework, C#, MVC, JavaScript/JQuery/HTML/CSS',
 'MY SQL, MongoDB, Data Warehouse Architecture, Database Architect, NoSQL, Python, ETL, HDF, Financial Services, Data Storage',
 'Machine-Learning (INDUSTRY only not school), Production Quality Code, Data Science, Mentoring others, NLP (NOT essential nice to have), AI (NOT essential nice to have)',
 'Machine-Learning, Artificial Intelligence, Scala, Java, Python, AWS, Cassandra',
 'Machine-Learning, Big-Data analytics, Java, Python, NLP',
 'Machine-Learning, Big-Data, Tensorflow, Speech Rec, AWS',
 'Machine-Learning, Customer Facing, SPARK, Predictive Modeling, Industrial IoT Analytics, Java/ Scala, Natural Language Processing (NLP), Deep Learning',
 'Machine-Learning, Data Science, Natural Language Processing, Python, Java, Scala, Solr, SPARK, Hadoop',
 'Machine-Learning, Data Science, Natural Language Processing, SPARK, Hadoop, Python, Java, Scala, Text Mining',
 'Machine-Learning, Deep Learning, Data Science, neural networks, Tensorflow, Reinforcement Learning, CNN, RNN',
 'Machine-Learning, Deep Learning, Java',
 'Machine-Learning, Deep Learning, Python C++ OR Java, Cyber Security, networking technologies, Big-Data, Data Science, Audio/Noise/Digital Signal, Ability to attain SECRET security clearance, Full Scope Poly',
 'Machine-Learning, Hadoop, Data Mining, R, Python',
 'Machine-Learning, Information Retrieval, Data Science, Big-Data Mining, Natural Language Processing (NLP), Text Mining',
 'Machine-Learning, ML Algorithms, Python, SQL, scale, SPARK, Hadoop, Applied statistics, Data Processing, Data Analysis',
 'Machine-Learning, NLP, Deep Learning, AI, Big-Data, Predictive Analytics, Python, R',
 'Machine-Learning, Natural Language Processing, Spark/Hadoop, Data Science, Data Engineer',
 'Machine-Learning, Predictive Analytics, Big-Data',
 'Machine-Learning, Python, Applied Mathematics',
 'Machine-Learning, Python, NLP Frameworks',
 'Machine-Learning, Python, R',
 'Machine-Learning, Python, R, Data Mining, Data Analytics',
 'Machine-Learning, Python, R, SQL, Data Mining, Hadoop, Statistical Analysis',
 'Machine-Learning, Python, Scala, or Java, Hadoop, Big-Data, R/Matlab, C++',
 'Machine-Learning, Python, Tableau, Hadoop, R, Large Data Set Organization & Analytics, Statistical Modeling, Algorithmic development, Data Visualizations & Reporting, Automation',
 'Machine-Learning, Python, pandas, numpy, Scipy, sklearn, Medical Informatics, Apache Spark',
 'Machine-Learning, SPARK, Scala, C++, Python, SQL, MPP, AWS',
 'Machine-Learning, Scala, Search Intent Optimization',
 'Machine-Learning, Tensorflow, Statistical/Machine-Learning Algorithms',
 'Magento, Magento 2, PHP, JavaScript, XML, JSON, SQL, OOP PHP, HTML, CSS',
 'Managing Accountants, Advanced Excel Skills, Real Estate Accounting, Property Management Accounting, Accounting Principles, General Ledger, Business Intelligence, GAAP, Advanced SQL skills, Data Analysis',
 'Master Data Management, PostgreSQL, SQL programming, Scripting Languages (Python/Ruby/Perl), Linux, AWS, ElasticSearch',
 'Matlab, SimuLink, LabView, Python, Automotive Diagnostics/Repair Experience, Data Processing, Raptor-Dev Tool Kit, Fuel Cell, Automotive, Model-based software',
 'Microsoft Dynamics 365, Microsoft TFS, Dynamics AX Implementations, full lifecycle implementation, Dynamics AX Certification, Data Modeling, Visual Studio',
 'Microsoft Excel, PowerPoint, Data Management, Reconciliations, collection process, Revenue Reporting, commission recording',
 'Mixed-Signal ASIC Development, Analog Development, Analog Front end Development, Data Converters A/D and D/A Blocks, bias blocks, precision oscillators, Silicon Design, Cadence tools, Biometric Algorithms',
 'Mobile Equipment Low Voltage Systems (12/24 volt), Creating Bill of Materials (BOMs)/SAP Data, High Voltage Applications, Class B Drivers License (PLUS), Electrical Schematics and harness drawings, AutoCAD Electrical 2015 software, 3D CAD Modeling, SolidEdge (PLUS), Cross-Functional Team Environment',
 'Modern/Idiomatic Web Dev - Python and JavaScript, Flask or Django, React/HTML/CSS, Relational Databases, Web Application Security/Data Protection, RESTful APIs, Git/Grunt/Gulp/Webpack/CI/CD, Google Cloud/AWS, Cybersecurity/Newtorking',
 'Modern/Idiomatic Web Dev - Python, Flask or Django, Relational Databases, Web Application Security/Data Protection, RESTful APIs, Git/Grunt/Gulp/Webpack/CI/CD, Google Cloud/AWS, Cybersecurity/Newtorking, Machine-Learning - Bonus',
 'Network Operations Center Engineering, Juniper Junos, Cisco IOS, NOC, Windows Server 2012, Linux Administration',
 'Networking, Cloud Computing, Routers, Switches, Network Security',
 'NoSQL, Ruby On Rails, XML and JSON based data handling and related APIs, AWS, Development for Android and/or iOS, JIRA, Jasper reporting server, Hadoop and/or MongoDB, .NET and/or Java, Agile-based development',
 'Node.js, Golang, NPM, Socket.io, High availability applications, Object-oriented Programming, Javascript Core Concepts, Prototypal Inheritance, Data Object Model, Data Structures',
 'Node.js, JavaScript, SPARK, NoSQL',
 'Node.js, REACT, Angular, JavaScript',
 'Node.js, React.JS, AngularJS',
 'OOP (Java/Python/C++ or similar), PHP, JQuery/Angular.js/Backbone.js',
 'Objective-C, iOS Development, Cocoa, Core Data, Core Animation, API',
 'Operations Analyst, Data Analysis, SQL, Excel',
 'Oracle, Oracle E-Business Suite, Oracle Forms, Oracle reports, bi publisher, PL/SQL, Oracle Applications Framework',
 'PHP, Java, Hadoop, Kafka, AWS, Python',
 'PHP, Ruby On Rails, AWS and Heroku, MVC architecture, JSON-Based APIs, Databases and data modeling/design techniques, GIT, Linux Operating Systems',
 'PID contoller, AC/DC/VFD motor drives, single/multi loopcontrollers, industrial automation instrumentation &maintenance, PLC Programming, HMI software programming / Human Machine Interface, electrical & mechanical system torubleshooting, Wonderware / data historian, Controls Engineering, AutoCAD',
 'Pattern Recognition, Predictive Modeling, NoSQL datastores, Hadoop / Pig / Spark / Hive, Statistical/Machine-Learning Algorithms, Hadoop / MapReduce / R / SAS / SPSS, Manufacturing',
 'Performance Testing, Linux, Python or Ruby, Distributed Systems',
 'Performance Testing, Linux, Python or Ruby, Distributed Systems, Performance Analysis and Benchmarking, Performance Engineer',
 'PostgreSQL, Java, Distributed Systems, Core Java Libraries, Unit Testing, Big-Data, File Systems, Ruby Scripting, Networking - SOAP - REST- HTTP- TCP/IP, SVN, or Git',
 'Presales Engineer, Infrastructure, Data Management, Storage, Hybrid Cloud Management',
 'Private Ethereum Compatible Blockchain, Ethereum Public Chain, solidity smart contracts, Go / Javascript / Rust / Python / C / C++ / Ruby, Mist / Geth / Truffle / Web3.js / Parity, ERC20 compatible token project, Distributed Clouds / Containerized Environments, Unix / Linux, GIT / GIT rebase workflow, Open Source Projects',
 'Product Development, Web based analytics/BI/Visualization, UI, JavaScript, CSS, HTML5',
 'Product Management, Business Intelligence, Product Roadmapping, Analytics, Quality Assurance, Software Development Methodologies, Release Management, Product Delivery',
 'Product Management, Enterprise B2B Platforms, SaaS, Cloud Computing, Business Intelligence, New Product Launches, Scaling Software Products',
 'Product Marketing, Enterprise Software, RPA Robotic Process Automation, Business Process Automation, Market Intelligence, Sales Enablement',
 'Product Marketing, Product Manager, Business intelligence/ analytics, Customer Facing, Solution Marketing, Solution Engineering, Big-Data',
 'Product Positioning, Scheduling, Analyzing Data, Land Acquisition, Marketing Analyst, Sales, Logistics/Supply Chain, Home Building, Custom homes',
 'Public Accounting, Biotech/Pharma Industry, Budget Planning, Budget monitoring, preparing proposals and contracts, Data Analysis',
 'Python, AWS, Big-Data Solution',
 'Python, AWS, Big-Data Solution, RESTful APIs',
 'Python, AWS, ETL, Mapreduce, Redshift, Lambda, Looker, Data Engineer',
 'Python, AWS, PHP, Hadoop, Cassandra, cloud platforms, MongoDB, NoSQL, ROR, Back-End',
 'Python, AWS, Redshift, ETL, Hadoop, SPARK, Hive, Kafka',
 'Python, C++, Big-Data, Kafka, SPARK, Docker, Containers',
 'Python, C++, Scala, SPARK, TDD / BDD, JavaScript, Large-Scale big data applications, Financial Services / Capital Markets, Unix Command Line, Data Structures',
 'Python, C++, TDD / BDD, JavaScript, Large-Scale big data applications, Financial Services / Capital Markets, Unix Command Line, Data Structures, Data Feed, Object-oriented Programming',
 'Python, Data Engineering, Data Systems (Spark/Hadoop/Hive/Pig/Storm/Elastic), Microservices architecture',
 'Python, Data Engineering, Data pipelining, AWS or Cloud',
 'Python, ETL, AWS',
 'Python, Java, Go, Unix, Linux, Cloud, AWS',
 'Python, Java, SQL (BigQuery/Oracle and Hive), Dimensional data modeling & schema design in DW, ETL design (Informatica/Talend/Pentaho), Map Reduce or MPP system, Write abstracted reusable code components, Design data warehouse and data pipelines, Architect/Build/Launch new data models, API Development',
 'Python, Java, SQL (BigQuery/Oracle and Hive), ETL design (Informatica/Talend/Pentaho), Write abstracted reusable code components, Design data warehouse and data pipelines, API Development, Linux Shell Scripting, Amazon Redshift and RDS Aurora, Data modeling design and documentation',
 'Python, JavaScript, Backbone.JS, JQuery, Angular.js, UI/UX, Big-Data',
 'Python, JavaScript, Django/Flask/Pyramid, REACT, Angular, Node.js, NoSQL, AWS/EC2, Google App Engine, CSS/HTML',
 'Python, Machine-Learning, neural networks, Risk Analysis, Econometrics, Portfolio Management, Equities, ETFs, NLP',
 'Python, Management, Machine-Learning, Statistics, Data Science',
 'Python, Multi-threaded Programming, asynchronous programming, REST APIs, SQL, Big-Data, Hadoop Ecosystem, NoSQL, SPARK, Kafka',
 'Python, MySQL, Pentaho',
 'Python, SAS, GLM, Actuarial Science',
 'Python, SPARK, ETL',
 'Python, SPARK, Hadoop, ElasticSearch, Cascading, Hive, AWS, Docker, RESTful, JSON',
 'Python, SQL, C/C++, Marketing Analytics',
 'Python, SQL, Tableau',
 'Python, SQL, Tableau, SPARK, hBase, Kafka, Statistical/Mathematical Modeling, NoSQL, scikit-learn, Hadoop',
 'Python, Scala, Unix Command Line, Large-Scale big data applications, Kafka, HDFS, Version Control, Containers, Orchestration, CI/CD Pipelines',
 'Python, closure, decorator, Data Structures, JSONBuddy, JSON Ana, NOSE Testing, Pie Testing, Unit Testing, API Testing',
 'Python/R/Julia, Formulating metrics for optimizing healthcare, Experiment design for client facing optimizations, Developing Model/Data driven solutions, Analysis & Modeling of Large Complex Datasets, Predictive Modeling and Analytics, Client Facing Roles, Machine-Learning',
 'Python/R/Julia, Predictive Modeling and Analytics, Machine-Learning, Big-Data - Hadoop/Pig/Hive/Similar, Visualization/Tableau, Healthcare Analytics',
 'Quality Control, Database Management Systems, Electronics Manufacturing, Production Data, Quality Engineering, Data Table Making, Visual Basic, Circuitry',
 'Quantitative Analysis / Analytics, Statistical Analysis, R/Python, SQL, Machine-Learning',
 'Queries, Scripting, Data Warehousing, Cloud, SQL Server, ETL, Ssis, SQL, MongoDB, Cosmos DB',
 'R Language, Python, Datasets, SQL Queries',
 'R, Python, SQL',
 'RUBY, Data Modeling, Whiteboarding, Test Driven Development, JS Frameworks',
 'RUBY, JavaScript, Node.js, REACT, MongoDB, PostgreSQL, MySQL, GIT',
 'RUBY, MongoDB, JavaScript, Node.js, REACT, PostgreSQL, MySQL, GIT',
 'RUBY, Web Application Development, Data Modeling, User Interface, Workflow, REACT, Redux, E-Commerce, Agile',
 'Rails, Angular.js, Data Modeling, Web Technologies, Advertising',
 'Rapid Application Development, wire-framing, Mocking, Prototyping, Excel/VBA/Python, Bloomberg Terminal (Eikon is a plus), Market Data API, Equity Derivatives, C/C++ or Java/Scala or C#/F#, Data Science Stack',
 'React.JS, Redux, MVC Javascript Development, RESTful API, 3rd Party API Integration, Node.js, graphql, Docker, Kubernetes',
 'Redshift, Vertica, Mapreduce, Pig, Hive, SPARK, Kafka, Kinesis, Storm',
 'Reporting, Architecture, Security',
 'Routers, Firewalls, Switches, Wireless Controllers and Access Points, Web Filtering and IPS, VOIP (Avaya), Cisco, SNMP Network Monitors, Network Security, Layer 1 Cabling',
 'Ruby On Rails, AWS, PHP',
 'Ruby On Rails, Scalability, Security, Algorithms, Big-Data Technologies (Hadoop/MapReduce/Spark)',
 'Ruby on Rails / RoR / Ruby / Rails, Data Structures, Algorithms',
 'SDET experience (work not academia), Developing Automated Integration/Unit Tests, Test Cases, Python, Linux, Systems Level, Automation Testing, Cloud environments, Big-Data Environments',
 'SDLC Methodologies, Oracle R12 business Suite, PL/SQL, ETL, Oracle data integrator (ODI), Informatica PowerCenter, OBIEE/Essbase, BI toolsets',
 'SOA experience on the backend, C/C++, Cross platform integration and analysis expert, Java/Javascript, Key IAB standards (VAST/VPAID/VMAP/MRAID/OpenRTB), Developing ad platforms, Large scale data analysis tools (Hadoop), Relational and non-relational database experience, Algorithms and Data Structures, Advertising ecosystems/ad servers/DSPs/SSPs',
 'SPARK, Airflow, Redshift, Python',
 'SPARK, Kafka, Druid',
 'SPARK, Kafka, Hadoop, Hive, Airflow, Java, Scala, Python, Azkaban, SQL',
 'SQL Queries, Marketing Analysis, E-Commerce',
 'SQL Server Development, TSQL, SSIS/SSRS, Data Warehousing and Data Modeling, C#/OOP',
 'SQL Server, Database Design, Ssis, Big-Data tools/technologies',
 'SQL Server, Dba, T-SQL, Performance Tuning, Debugging, C#, OLTP, DM/DW, SSIS/SSRS/SSAS, Big-Data / No SQL',
 'SQL Server, PL/SQL, T-SQL, SQL Queries, Data Integration, Data Analysis, Problem Solving Skills',
 'SQL Server, SSIS/SSRS, Data Scraping, Microsoft Office 365, Power BI, SharePoint, Oracle EBS, ETL Process, OBIEE, Hyperion',
 'SQL, Data Analysis, T-SQL, Writing SSRS Reports, Stored Procedures, Table/Scalar Value Tables, SSIS/SSAS, MS Excel',
 'SQL, Data Modeling, Data Analytics & Visualization, ETL',
 'SQL, Data Modeling, Performance Tuning, Disaster Recovery, AWS, MySQL, PostgreSQL, ERD, Redshift',
 'SQL, Data, SQL Server, MySQL',
 'SQL, Database Architect, MySQL',
 'SQL, Python, R, Statistics, Analytics, E-Commerce, Web Analytics, Subscription Media, Business Intelligence, Business Analytics',
 'SQL, Python, R, Tableau, DOMO, Looker, Hive, SPARK, AIB testing, E-Commerce',
 'SQL, R, Python, Frequentist, Bayesian',
 'SQL, Ssrs, SSAS, Ssis',
 'SQL, Tableau, Data Analyst, Ssrs, Ssis, adhoc',
 'SQL, Visual Basics, Excel, Access',
 'SaaS, Service-Oriented Architecture, micro-services, Management, Distributed Java Systems, NoSQL, Java, Spring, AWS',
 'SaaS, Service-Oriented Architecture, micro-services, hands-on development, Management, Distributed Java Systems, Cassandra, NoSQL',
 'SaaS/IaaS, Mac, Active Directory, AWS, Network',
 'Sales Engineering, IOT, Data Engineer- Hadoop/spark/kafka, Cloud- Aws/azure, Python/R/Scala/SAS',
 'Sales Representative, Business Development, Software Sales, Mobile Solutions, Mobile Software',
 'Salesforce Certification, Workflow rules / Process Builder, Automate Dashboards, Segment Data, Apex Coding, Marketing Cloud Analytics, Worked in or with Sales Team, Conduct Training / Webinars, Leadership',
 'Scala /Python /Java, Algorithms, Hadoop, SQL, Data Analysis',
 'Scala, Java, RESTful API, Microservices, Amazon AWS',
 'Security Auditing methodologies, PCI Data Security Standards, Information Security, HIPAA, SIEM, Penetration Testing, Firewalls, active directory security',
 'Server-side C++, Linux, GPU, IPC, Asynchronous Dataflows, Real-Time Low Latency applications, Lock-free Data Structures, Algorithms, Accelerators',
 'SharePoint, .NET, Power BI, Microsoft Teams, Microsoft Technology Stack, system design, Application Databases, Cloud Computing, Batch/Shell scripting',
 'SharePoint, .NET, SDLC, Power BI, Microsoft Teams, Great Plains/Dynamics, Requirements Gathering, Mobile Development, Publishing, Telecom and/or Broadband',
 'Smack Stack, Scala, Amazon Web Services (AWS)',
 'Software Development in C++, Python, Linux, Algorithm Development, Data Structures, Boost/STL',
 'Software Development in C++, Python, Linux, Cloud environments, Big-Data Environments',
 'Software Development, Business Intelligence, C#, .NET, Java',
 'Software Development, Python, Java, Linux, AWS/EC2/S3, MySQL or PostgreSQL, TS/SCI w/ Polygraph, Machine-Learning, Artificial Intelligence, Hadoop',
 'Software Development, SQL, Data Analysis, Excel, Data Analyst, Python, Verbal and Written, Financial Analysis, analytical reports, Planning',
 "Software Engineer, C++(OOD) C++ programming, Algorithm Design/development, Data Structures, Program Optimization, Customer Facing UI Design, GPU's, Redhat, multi-threaded design",
 'Software Engineer, C, C++, Python, Linux Kernel, Drivers, ARM, 686, and/or RISC-V',
 'Statistics, Machine-Learning, Applied Mathematics',
 'Statistics, Mathematics, Computer Science, Machine-Learning, Product Development, Python, data architectures',
 'Systems Programming, C++, Python, Object Storage, Object Store, Data Storage, Network File System, Storage Programming, REST Architecture, Amazon Web Services (AWS)',
 'T-SQL, SQL, SQL Server, C#, XML, Database Design, Data Analysis',
 'TS/SCI with FS Polygraph Clearance, Java, Map Reduce, Eclipse, Pig, Hadoop, Maven',
 'Tableau, ETL, SQL, Automation Technologies, Unix/Linux, Python/R/equivalent',
 'Technical Writing - Software Documentation, Create user & functional requirements, Create test plans and test scripts, Design materials (mock-ups/process flowcharts), Create use cases to perform task workflow analysis, Finance & accounting concepts, Salesforce.com, Excel functions to manipulate and visualize data, Agile/Scrum and Kanban environments',
 'Telecommunications Industry, Sales Engineering, Sales Consulting, Pre-Sales Support, Application Design, Complex Enterprise Voice & Data Products/Services, Customer presentations, Network Design, Fiber-to-the-Premise (FTTP) Technology, Cost Estimates for Fiber Optic Drops',
 'Test Automation, Java/Javascript, Selenium Web Driver / TestNG, Algorithm & Data Structure Development',
 'Test Automation, Selenium, API Testing, Ruby/JavaScript/Python, Linux, HTTP, Public Clouds- AWS/EC2/AZURE',
 'Test Automation, Selenium, API Testing, Ruby/JavaScript/Python, Linux, HTTP, Public Clouds- AWS/EC2/AZURE, Distributed Systems',
 'Top Secret Clearance with CI or FSP, Java C & C++, Cloud Engineering, Big-Data Storage (NoSQL) Hbase/Big Table/CloudBase, Hadoop Distributed File System( HDFS), Serialization JSON or BSON, Windows (2000, 2003, XP, Vista), UNIX or Linux (RedHat 3-5), software integration and software testing, Implementing web services (SOAP and RESTful)',
 'UAV, Drone, Mechanical testing, Soldering, data acquisition, Electro-mechanical sub-assemblies/assemblies, AutoCAD, LabView, 3D CAD, SolidWorks',
 'UML, Hadoop/Pig/Hive, SQL/NoSQL, EC2/S3/DynamoDB, Data Security',
 'Web Application Development, HTML, CSS, JavaScript, PHP, SQL, .NET, Node.js, Data Visualization / Imaging, Agency Experience',
 'Widget design, JavaScript, UX, HTML5, CSS3, REST API, UI, Enterprise Products, REST APIs',
 'Writing Algorithms, C++, Python, Java, Data Interpretation, LabView, Matlab, Communication Interfaces, JavaScript, Firmware',
 'XML, Project Management, Data Analysis',
 'analyze and interpret complex digital data, Database, date wrangling, Data Visualization',
 'applications Mesos-Kafka-Spark-Yarn-Ignite, Striim-ScaleOut-Hazelcast-Tinker Pop-Blaze Graph, SciKit-Tensorflow-Torch-Café-Cassandra-Hadoop, Mongo-MySQL-Oracle-MemSQL',
 'cloud architecture, AWS Lambda, Messaging technologies, Data Processing Pipelines, Redis',
 'econometric modeling, Statistical Modeling, Quantitative research methods, R, Python, SPARK, SQL, Machine-Learning Algorithms, Machine-Learning, Data Mining',
 'fullstack, Node.js, AWS',
 'iOS App Development, iOS frameworks and SDK, Objective C, SWIFT, RESTful APIs/JSON, Core Data Frameworks, XCode/AppCode, MySQL/SQLite, Context-Aware Application & API Builds, React Native experience or knowledge (Huge Plus)',
 'modeling, Python, Panda, applications'}

def wordnet_pos(tag):
    """Map a Brown POS tag to a WordNet POS tag."""
    
    table = {"N": wordnet.NOUN, "V": wordnet.VERB, "R": wordnet.ADV, "J": wordnet.ADJ}
    
    # Default to a noun.
    return table.get(tag[0], wordnet.NOUN)

def newtext(text):
    '''Return a new text after the stopwords and lemmatizing'''
    blob = TextBlob(text)
    new_text = " ".join(w for w in blob.words if w.lower() not in stop_words)
    blob = TextBlob(new_text)
    tags = [wordnet_pos(x[1]) for x in blob.pos_tags]
    new_text = " ".join(x.lemmatize(t) for x, t in zip(blob.words, tags))
    blob = TextBlob(new_text)
    return new_text

def ReturnCount(text):
    '''Return a word counts dictionary'''
    new = newtext(text)
    blob = TextBlob(new)
    return blob.word_counts  

def countdata(text):
    '''
    Sort the counts of each distinct word
    input:text(string)
    output:datafrane of the word counts dictionary
    '''
    dic = ReturnCount(text)
    count=pd.DataFrame(list(dic.items()), columns=['word', 'count'])
    return count

def countsort(text):
    '''
    sort the countdata
    input:text(string)
    output:dataframe of 10 most frequent words and their counts
    '''
    df1 = countdata(text)
    newdf = df1.sort_values(by='count', ascending=False).head(20)
    return newdf

def barplot(text,theme):
    '''
    input:text(string),theme like art,sports(string)
    output:barplot of the most frequent words and counts
    '''
    data = countsort(text)
    new = newtext(text)
    plt.rcParams['figure.figsize'] = (10, 10)
    sns.set(font_scale = 1.5)

    sns.set_style("whitegrid")
    fruit_bar = sns.barplot(x = "word", y = "count", data =data)
    for item in fruit_bar.get_xticklabels():
        item.set_rotation(60)
    plt.title(theme)
    fruit_bar.set(ylabel = 'Count', xlabel = 'Word')
    p=plt.show()
    return p

def wordcloud(text,theme):
    '''
    input:text(string),theme like art,sports(string)
    output:wordcloud of the text
    '''
    data=countsort(text)
    new=newtext(text)

    # Generate a word cloud image
    wordcloud = WordCloud().generate(new)

   # lower max_font_size
    wordcloud = WordCloud(max_font_size=40).generate(new)
    plt.figure()
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
    plt.title('wordcloud of '+theme)
    p=plt.show()
    return p

Top Skills Overall (Data Scientist, Data Engineer, Data Analyst, Business Intelligence)¶

skills=' '.join([str(i) for i in skills_df['Skills']])
skills_frequency = countsort(skills)
skills_frequency = skills_frequency.sort_values(by = 'count', ascending = False).reset_index(drop = True)
skills_frequency

(ggplot(skills_frequency, aes(x = "word", y = 'count', fill='word')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Overall Skills - Data Scientist, Data Engineer, Data Analyst, and Business Intelligence", x = "Skills", y = "Frequency")
)

<ggplot: (-9223372029308767842)>

def grey_color_func(word, font_size, position, orientation, random_state=None, **kwargs):
    """
    Purpose : Change the color of the words in the word cloud to grey scale 
    """
    return "hsl(0, 0%%, %d%%)" % random.randint(70, 100)

overall_skills = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
overall_skills.generate(skills)
plt.figure(figsize = (15,7.5))
plt.imshow(overall_skills.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Overall Skills - Data Scientist, Data Engineer, Data Analyst, and Business Intelligence")
plt.show()

ds_skills

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-516-f984fb03098e> in <module>()
----> 1 ds_skills

NameError: name 'ds_skills' is not defined

Top Skills - Data Scientist¶

ds_skills=' '.join([str(i) for i in data_scientist_df['Skills']])
ds_skills_frequency = countsort(ds_skills)
ds_skills_frequency = ds_skills_frequency.sort_values(by = 'count', ascending = False).reset_index(drop = True)
ds_skills_frequency

(ggplot(ds_skills_frequency, aes(x = "word", y = 'count', fill='word')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Overall Skills - Data Scientist", x = "Skills", y = "Frequency")
)

<ggplot: (-9223372029285089550)>

ds_overall_skills = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
ds_overall_skills.generate(ds_skills)
plt.figure(figsize = (15,7.5))
plt.imshow(ds_overall_skills.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Overall Skills - Data Scientist")
plt.show()

Top Skills - Data Engineer¶

de_skills=' '.join([str(i) for i in data_engineer_df['Skills']])
de_skills_frequency = countsort(de_skills)
de_skills_frequency = de_skills_frequency.sort_values(by = 'count', ascending = False).reset_index(drop = True)
de_skills_frequency

(ggplot(de_skills_frequency, aes(x = "word", y = 'count', fill='word')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Overall Skills - Data Engineer", x = "Skills", y = "Frequency")
)

<ggplot: (7551618287)>

de_overall_skills = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
de_overall_skills.generate(de_skills)
plt.figure(figsize = (15,7.5))
plt.imshow(de_overall_skills.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Overall Skills - Data Engineer")
plt.show()

Top Skills - Data Analyst¶

da_skills=' '.join([str(i) for i in data_analyst_df['Skills']])
da_skills_frequency = countsort(da_skills)
da_skills_frequency = da_skills_frequency.sort_values(by = 'count', ascending = False).reset_index(drop = True)
da_skills_frequency

(ggplot(da_skills_frequency, aes(x = "word", y = 'count', fill='word')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Overall Skills - Data Analyst", x = "Skills", y = "Frequency")
)

<ggplot: (7551532345)>

da_overall_skills = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
da_overall_skills.generate(da_skills)
plt.figure(figsize = (15,7.5))
plt.imshow(da_overall_skills.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Overall Skills - Data Analyst")
plt.show()

Top Skills - Business Intelligence¶

bi_skills=' '.join([str(i) for i in BI_df['Skills']])
bi_skills_frequency = countsort(bi_skills)
bi_skills_frequency = bi_skills_frequency.sort_values(by = 'count', ascending = False).reset_index(drop = True)
bi_skills_frequency

(ggplot(bi_skills_frequency, aes(x = "word", y = 'count', fill='word')) +
 geom_bar(stat = "identity", position='stack', show_legend = False)+
 theme(figure_size=(10, 6),axis_text_x= element_text(rotation=80))+
 labs(title = "Overall Skills - Business Intelligence", x = "Skills", y = "Frequency")
)

<ggplot: (7563990262)>

bi_overall_skills = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
bi_overall_skills.generate(bi_skills)
plt.figure(figsize = (15,7.5))
plt.imshow(bi_overall_skills.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Overall Skills - Business Intelligence")
plt.show()

3.) Analysis - Descriptions¶

description_df = master_data.copy()

#generate wordlist and lemmatizer
wordnet_lemmatizer = WordNetLemmatizer()

#preprocessing function
def my_preprocessing(s):
    s = s.lower() # downcase
    s = re.sub(r"\W"," ",s)
    s = re.sub(r"\d"," ",s) 
    s = re.sub(r"\s+[a-z]\s+"," ",s)
    s = re.sub(r"^[a-zA-Z\s*]*what you will be doing",'',s)
    s = re.sub(r"\s+",' ',s)
    return s

responsibilities=[]
benefits=[]
jddataset = description_df['Description']
for i in range(len(jddataset)):
    s = my_preprocessing(jddataset[i])
    if 'what in it for you' in s :
        responsibility = s.split('what in it for you')[0]
        benefit = s.split('what in it for you')[1]
    else:
        responsibility = s
        benefit = ''
    responsibilities.append(responsibility)
    benefits.append(benefit)

def my_tokenize(s):
    tokens = nltk.tokenize.word_tokenize(s) # split string into words (tokens)
    tokens = [t for t in tokens if len(t) > 2] # remove short words, they're probably not useful
    tokens = [wordnet_lemmatizer.lemmatize(t) for t in tokens] # put words into base form
    tokens = [t for t in tokens if t not in set(stop_words)]# remove stopwords
    s = ' '.join(tokens)
    return s

#tokenize responsibilities
for i in range(len(responsibilities)):
    responsibilities[i] = my_tokenize(responsibilities[i])
    i = i+1

text = pd.DataFrame(responsibilities,columns = ['re'])
text['re']

0      traffic predictor risk assessment tool determi...
1      iterate improve risk engine fraud modelswork i...
2      become cornerstone company capability across p...
3      scientist engineer responsible following build...
4      scientist please read located north austin dow...
5      working help build cool product primary respon...
6      descriptiondata scientist need analyze financi...
7      scientist engineer responsible following build...
8      scientist mining set statistic philadelphiawe ...
9      scientist machine learning python tableau tren...
10     scientist southbaywe creating platform product...
11     work scientist product manager customer succes...
12     ideal candidate expertise working large set ex...
13     develop implement advanced statistical perform...
14     scientist report chief technology officer obje...
15     ideal candidate expertise working large set ex...
16     develop implement advanced statistical perform...
17     develop nlp help customer understand contracts...
18     responsible researching building machine learn...
19     develop custom algorithm apply set evaluate ef...
20     traffic predictor risk assessment tool determi...
21     combine understanding goal machine learning co...
22     senior scientistwe looking scientist part inst...
23     senior scientistwe looking scientist part inst...
24     san jose relocation included application consi...
25     inform product strategy influence product road...
26     senior scientistwe looking scientist part inst...
27     looking scientist part instructional team seek...
28     looking scientist part instructional team seek...
29     creating develop launch next generation financ...
30     creating develop launch next generation financ...
31     growing team adapting analyst teaching python ...
32     growing team adapting analyst teaching python ...
33     creating develop launch next generation financ...
34     creating develop launch next generation financ...
35     senior scientist machine learning python table...
36     chief scientist machine learning python philly...
37     chief scientist machine learning python bay ar...
38     creating develop launch next generation financ...
39     cross functional team designer developer produ...
40     center innovative corporate group rapidly grow...
41     key team member innovate smart manufacturing p...
42     stellar scientist hadoop platform read interna...
43     frame conduct experiment drawing tremendously ...
44     relocation nashville availablelocated nashvill...
45     looking chief scientist responsible establishi...
46     creating machine learning solution assist deli...
47     creating machine learning solution assist deli...
48     working help build cool product primary respon...
49     develop new machine learning cognitive computi...
50     role bring passion technology apply skill plat...
51     role bring passion technology apply skill plat...
52     working alongside small team elite engineer pu...
53     role bring passion technology apply skill plat...
54     position responsible mining demographic clinic...
55     working alongside small team elite engineer pu...
56     using complex statistical improve portfolio pe...
57     conduct research meta learning machine learnin...
58     growing team adapting analyst teaching python ...
59     work exciting problem recommendation improve u...
60     scientist working closely client team build pr...
61     part decision function including contributing ...
62     scientist python please read based san mateo w...
63     year intelligence communityexperience verifica...
64     year intelligence communityexperience verifica...
65     construct end end flow raw prediction crunch a...
66     seeking someone build support critical backend...
67     inform product strategy influence product road...
68     expert convolutional neural network tensorflow...
69     scientist python please read based san mateo w...
70     construct fully automated risk control engine ...
71     advanced emerging technology team currently ne...
72     assist designing creation debugging sql etl pa...
73     engineer looking new opportunity raleigh area ...
74     engineer looking new opportunity raleigh area ...
75     engineer looking new opportunity raleigh area ...
76     sql nosql database determining requirement pla...
77     san jose relocation included application consi...
78     designing maintaining improving warehouseact f...
79     maintain improve pipeline using hadoop spark a...
80     engineer please read mobile company us big pre...
81     engineer please read mobile company us big pre...
82     mysql warehouse technical ownership maintenanc...
83     creating curating bigtables hbase cassandra st...
84     provides consultation technical expertise rega...
85     san jose relocation included application consi...
86     senior engineer posse strong python building i...
87     apply best principle agile methodology researc...
88     build maintain big platform using open source ...
89     help scale complex machine learning pipeline u...
90     lead team engineer build scalable highly avail...
91     lead strategy implementation storage access pl...
92     lead strategy implementation storage access pl...
93     looking strong engineer contribute creating wo...
94     integral part web optimization product team ch...
95     senior engineer experienced warehousing year t...
96     partner digital product customer supply chain ...
97     assemble large complex set meet functional non...
98     coding daily language like python scala integr...
99     coding daily language like python scala integr...
100    identify define algorithm improvement trading ...
101    identify define algorithm improvement trading ...
102    integral part web optimization product team ch...
103    build scalable reliable near real time pipelin...
104    big engineer please read title big engineersal...
105    integral part web optimization product team ch...
106    leading voice authority architecture newest pr...
107    uptime collection appropriate redundancy speci...
108    senior engineer etl sql big please read leadin...
109    exciting stuff big showing linux shell scripti...
110    frame conduct experiment drawing tremendously ...
111    senior engineer python tableau sql please read...
112    building product help customer fulfill service...
113    building product help customer fulfill service...
114    scale improve big platform provide operational...
115    lead maintenance scalable pipeline gather requ...
116    help develop analyze new predictive tool worki...
117    work cross functionally different department d...
118    use sql python linux aws environment support i...
119    engineer responsible following deployment new ...
120    developing connected automated vehicle infrast...
121    develop refine oversee master standard includi...
122    designing improving functionality web applicat...
123    big engineer responsible following architect d...
124    helping develop big analytical application des...
125    helping develop big analytical application des...
126    exciting stuff big showing yourlinux shell scr...
127    helping develop big analytical application des...
128    crucial contributor cloud backend architecture...
129    creating curating bigtables hbase cassandra st...
130    crucial contributor cloud backend architecture...
131    crucial contributor cloud backend architecture...
132    building working pipeline help manage understa...
133    build develop big nosql solution sql database ...
134    bring deep knowledge building real product usi...
135    senior warehouse developer etl sql big please ...
136    star schema need built improved reporting feed...
137    helping develop big analytical application des...
138    lead big engineer dod clearance please read go...
139    senior engineer developing web application jav...
140    senior engineer year professional industry ple...
141    background machine learning collaborate engine...
142    partner sale partnership gain deep understandi...
143    architect three year industry please read situ...
144    senior engineer please read million people yea...
145    develop implement hadoop based big solution di...
146    nutshell working ingest huge amount scale crea...
147    developer big please read located north austin...
148    processing large amount using big technology h...
149    build support service managing open identify r...
150    work current team stakeholder create innovativ...
151    work current team stakeholder create innovativ...
152    individual work architect develop release adva...
153    pipeline collection appropriate redundancy spe...
154    lead effort democratize access akuna architect...
155    essential responsibility center consulting eng...
156    essential responsibility center consulting eng...
157    essential responsibility center consulting eng...
158    key member gui team ensure success building ro...
159    role responsibility include engaging client nu...
160    experimentation platform like doe debating mer...
161    senior engineer developing web application jav...
162    develop object store client push pull wire spe...
163    develop object store client push pull wire spe...
164    infrastructure team mission build solution int...
165    develop large scale distributed convert day st...
166    java big pipeline engineer please read impleme...
167    develop linux smart client transfer parallel m...
168    develop large scale distributed convert day st...
169    build big processing streaming line processing...
170    rolewe got lot moving part developed house pur...
171    technical resource customer presales support s...
172    high performance developing multi threaded sha...
173    based san mateo right near closest train stop ...
174    architectural knowledge improve reliability ef...
175    role responsibility include engaging client nu...
176    member team help achieve goal designing develo...
177    member team help achieve goal designing develo...
178    year enterprise scale network operation year h...
179    rolewe got lot moving part developed house pur...
180    develop robust scalable core miso platform opt...
181    infrastructure team mission build solution int...
182    build application io platform collaborate team...
183    build maintain distributed processing componen...
184    company leader intelligence helm warehousing p...
185    develop maintain multiple database database pe...
186    located san jose ruby developer engineer pleas...
187    spearhead major infrastructure devops reliabil...
188    heavily involved architecture hand front end a...
189    full stack engineer posse strong python javasc...
190    maintain improve product offering location int...
191    sql developer understands complex code ha stro...
192    salesforce engineer deal highly ambiguous prob...
193    java client project working exciting innovativ...
194    work large commercial cad application perform ...
195    role actively participates database developer ...
196    function senior member team working responsibi...
197    develop java spring json jee rest work agile s...
198    participate entire lifecycle concept full stac...
199    develop java spring json jee rest work agile s...
200    participate entire lifecycle concept full stac...
201    responsible partnering implement database web ...
202    machine learning engineer work scientist devel...
203    machine learning engineer year please read app...
204    san jose relocation included application consi...
205    passionate sql engineer love solving problem r...
206    expand competitive edge delivering world class...
207    looking primarily etl focused developer aws cl...
208    help architect develop cloud based platform in...
209    help architect develop cloud based platform in...
210    join small highly collaborative team enjoy wor...
211    join small highly collaborative team enjoy wor...
212    working latest product involve automated prici...
213    integrate principal team comprised engineer bu...
214    building scalable high quality code solid code...
215    lead devops effort petabyte scale collection m...
216    based san mateo right near closest train stop ...
217    based san mateo right near closest train stop ...
218    build application io platform collaborate team...
219    supporting head operation quality control manu...
220    build configure platform service oriented arch...
221    senior android engineer please read relocate a...
222    coding daily language like python scala javasc...
223    looking variety expert expertise following are...
224    looking variety expert expertise following are...
225    work highly interactive team oriented environm...
226    position involve designing developing algorith...
227    primary responsibility position include analyz...
228    leading multi platform application analyze fea...
229    consumer financing company offering combinatio...
230    mentor lead junior developer opportunity use t...
231    mentor lead junior developer opportunity use t...
232    engineer responsible ensure feature deliverabl...
233    leading automation effort growing organization...
234    mentor lead junior developer opportunity use t...
235    critical member high performance team solving ...
236    critical member high performance team solving ...
237    building high quality application sweet modern...
238    leading automation effort growing organization...
239    lead new technology capability expansion effor...
240    backend engineer implement api endpoint using ...
241    create prototype develop execute machine learn...
242    building tool improve machine learning project...
243    fpga board level hardware engineer responsible...
244    full stack including front end visualization m...
245    code new user facing feature integrate various...
246    improving query speed setting best practice te...
247    helping create next big big need positionexper...
248    net developer asp net message looking highly s...
249    role work team research prototype build mainta...
250    work aspect information technology basic works...
251    work aspect information technology basic works...
252    coding daily language like python scala javasc...
253    working team expert golang developer developin...
254    full stack engineer posse strong python javasc...
255    full stack engineer posse strong ruby javascri...
256    work team building generate workload tool unde...
257    full stack engineer posse strong ruby mongodb ...
258    work team building generate workload tool unde...
259    salesforce developer deal highly ambiguous pro...
260    senior salesforce developer deal highly ambigu...
261    serve team frontend lead developer developing ...
262    help develop manage enforce related policy pro...
263    lead example take ownership face adversity abi...
264    looking experienced full stack web developer m...
265    collaborate exciting tech focused challenging ...
266    work within team solve complex problem daily b...
267    lead team building platform create driven dash...
268    engineer raleigh area read adtech startup base...
269    engineer raleigh area read adtech startup base...
270    building high performance massively scalable a...
271    role work within small talented imaging team u...
272    lead senior mobile engineer responsible follow...
273    senior mobile engineer responsible following i...
274    building high performance massively scalable a...
275    mixed signal circuit function top level analog...
276    role work within small talented imaging team u...
277    implementing automotive product part approval ...
278    role develop ship safety critical high perform...
279    work team support several enterprise applicati...
280    role work within small talented imaging team u...
281    lead platform engineer technical stack respons...
282    lead platform engineer technical stack respons...
283    developer building highly scalable web applica...
284    coding daily language like python scala javasc...
285    build maintain distributed processing componen...
286    responsibility may include installing troubles...
287    network strategy analyze price complex service...
288    seeking experienced developer help build scala...
289    act subject matter expert cisco cisco cisco un...
290    act subject matter expert cisco cisco cisco un...
291    based denver leading tech company ha revolutio...
292    develop restful api work closely team member d...
293    working thing sql queriesscriptingmigrationsda...
294    build architecture lead marquee platform code ...
295    mentor lead junior developer opportunity use t...
296    responsible creating open source application v...
297    collaborate exciting tech focused challenging ...
298    instrumental driving kafka implementation kafk...
299    contribute engineer team passionate delivering...
300    contribute engineer team passionate delivering...
301    platform engineer rail angular please read loc...
302    build application io platform collaborate team...
303    developing application solution reporting etc ...
304    lead senior mobile engineer responsible follow...
305    lead senior mobile engineer responsible follow...
306    building high performance massively scalable a...
307    lead manager technical stack responsible maxim...
308    department looking talented senior core java d...
309    identifies risk threat vulnerability network a...
310    develop core capability used build rich functi...
311    designing developing troubleshooting product d...
312    implement core component orchestration stack o...
313    lead conduct defensive cyberspace operation dc...
314    help architecture decision complex million lin...
315    linux developer please read immediate consider...
316    qualified candidate responsible service produc...
317    qualified candidate responsible service produc...
318    feature multi tenancy high availability scalab...
319    developer new feature functionality large web ...
320    creating clean well documented efficient stand...
321    feature multi tenancy high availability scalab...
322    partner product owner scrum master optimize im...
323    partner product owner scrum master optimize im...
324    improving query speed setting best practice te...
325    based san mateo right near closest train stop ...
326    maintain improve product offering location int...
327    working aspect deploying maintaining upgrading...
328    infrastructure architecture production environ...
329    job get opportunity exercise polished net skil...
330    building implementing high quality front end v...
331    maintain various component web application cre...
332    deploy support high performance highly availab...
333    deploy support high performance highly availab...
334    info coming soon need positionat least year kn...
335    role opportunity technical leader working smal...
336    use deep learning optimize large scale machine...
337    leading automation deployment monitoring aws h...
338    work team building generate workload tool unde...
339    leading automation deployment monitoring aws h...
340    implement automated part sldc ensure quality e...
341    build maintain distributed processing componen...
342    develop solidity smart contract run ethereum p...
343    responsibility build maintain distributed proc...
344    create sophisticated efficient visually pleasi...
345    product area lead effort one feature work team...
346    full stack web mobile application variety lang...
347    analyze develop high quality web application p...
348    typical primary task develop concept uav inclu...
349    bring technical skill developing professional ...
350    push limit innovation disrupt industry world c...
351    working product multi iteration product releas...
352    position involve designing developing algorith...
353    title mid level web developer javascript angul...
354    title mid levelcompensation negotiablecontract...
355    gather finalize requirement based product requ...
356    building high performance massively scalable a...
357    develop robust scalable core miso platform opt...
358    develop robust scalable core platform optimize...
359    leverage knowledge medical device participate ...
360    technical solution around cisco cisco focused ...
361    solving complex architecture working distribut...
362    work team develop requirement establish constr...
363    year strong understanding object oriented prog...
364    daily responsibility include architecture code...
365    serf technical expert head end monitor control...
366    provide senior level family multi tier azure c...
367    one successful technology start ups philadelph...
368    lead engineer please read immediate considerat...
369    architect centric networked rapidly changing g...
370    responsible creating driven web application wo...
371    looking add talented contract hire senior engi...
372    engineer play key role integral part team exci...
373    work platform team develop automated integrati...
374    work platform team develop automated integrati...
375    senior engineer performs professional work con...
376    manager authority stop production process feel...
377    join small highly collaborative team enjoy wor...
378    develop robust scalable platform optimize algo...
379    senior java programmer developing laboratory l...
380    work experienced staff develop deliver cutting...
381    responsibility position include helping implem...
382    work industry leading mobile application used ...
383    developing solution control mobile road constr...
384    developing solution control mobile road constr...
385    year strong understanding object oriented prog...
386    support lan wan network segment intranet surve...
387    work risk based pricing product part small tea...
388    work platform team develop automated integrati...
389    cloud engineer active sci fsp year including r...
390    member hybris team senior application engineer...
391    work platform team develop automated integrati...
392    work experienced staff develop deliver cutting...
393    believe highly hard work creativity innovation...
394    creating enhancing saas based solution create ...
395    maintain various component web application cre...
396    java client project working exciting innovativ...
397    java client project working exciting innovativ...
398    java client project working exciting innovativ...
399    believe highly hard work creativity innovation...
400    engineer responsible designing writing code ca...
401    support lan wan network segment intranet surve...
402    support lan wan network segment intranet surve...
403    developing client side view feature using reac...
404    exciting stuff big showing linux shell scripti...
405    working along side portfolio manager lead stra...
406    implement solution fully meet requirement lead...
407    serve member team implementation control hardw...
408    responsible improvement automated manufacturin...
409    deep learning experienced neural network tenso...
410    act electrical engineer complex electrical inf...
411    junior senior level position available review ...
412    developing leading edge controlled random veri...
413    developing leading edge controlled random veri...
414    building high performance massively scalable a...
415    building high performance massively scalable a...
416    develop maintain new existing work build core ...
417    believe way implementing case know designing i...
418    implementing technology solution client addres...
419    implementing technology solution client addres...
420    lead sourcing support advanced cloud computing...
421    analyst analyze hyper local source develop acc...
422    working side side client developing team provi...
423    obsession number desire tell story driven perf...
424    building maintaining complex marketing custome...
425    integral member corporate finance team positio...
426    lead analyst python tableau sql please read le...
427    summaryperforms subject matter expert rdms vis...
428    working large consumer set extracting analyzin...
429    creating enhancing tableau dashboard performin...
430    critically evaluate information gathered multi...
431    maintain enhance end user table reporting tool...
432    maintain enhance end user table reporting tool...
433    analyst please read solid global organization ...
434    lead analyst python tableau sql please read le...
435    senior analyst python tableau sql please read ...
436    database warehouse analyst involved warehouse ...
437    working highly confidential initiative drive d...
438    conducting support variety analytic solution c...
439    conducting support variety analytic solution c...
440    conducting support variety analytic solution c...
441    work engineer scientist analyst build extend t...
442    manage delivery report presentation recommenda...
443    collaborate technical service platform support...
444    extracting analyzing creating report searching...
445    process monthly capitation payment reporting p...
446    extracting analyzing creating report searching...
447    support month end close reporting support annu...
448    reporting analyst responsible providing timely...
449    salesforce analyst responsible following proac...
450    financial analyst position responsible reporti...
451    financial analyst position responsible reporti...
452    position report information officer responsibl...
453    position report information officer responsibl...
454    position report information officer responsibl...
455    successful candidate must show thrive excel wo...
456    using incident feature tracking create ticket ...
457    project control analyst pca assist project pro...
458    senior project control analyst lead pca assist...
459    seeking lead salesforce configurator analyst j...
460    seeking lead salesforce configurator analyst j...
461    interface corporate department facility person...
462    interface corporate department facility person...
463    looking experienced senior analyst responsible...
464    looking experienced senior analyst responsible...
465    preparation annual planpreparation monthly qua...
466    assisting annual market positioning study assi...
467    interface corporate department facility person...
468    creates maintains ensures quality assurance hu...
469    analyze develop high quality web application p...
470    managing team year related field integration w...
471    report developer please read strong global org...
472    report developer please read strong global org...
473    translating taking complex need join blend und...
474    hand director intelligence please read descrip...
475    hand director intelligence please read descrip...
476    conduct process need assessment preliminary co...
477    conduct process need assessment preliminary co...
478    responsible ownership intelligence including m...
479    responsibility include limited handle bodily i...
480    healthcare solution company orange county area...
481    ideal candidate working across unit site partn...
482    designing deploying scratch database database ...
483    lead build team deliver high impact insight fi...
484    based seattle intelligence saas platform compa...
485    etl tableau developer team working team develo...
486    hand leader engaging complex challenge informa...
487    developer primarily responsible programming ch...
488    responsibility overall relationship customer s...
489    oracle application developer read technology c...
490    responsible recurring reporting across build t...
491    must play key role growth tasked rapid expansi...
492    responsible designing documenting helping impl...
493    edward enterpriseone developer please read sol...
494    develop write code new existing database prima...
495    develop product solution content support outbo...
496    oversight requirement functional architecture ...
497    creates information solution manages analyst d...
498    edward enterpriseone developer please read sol...
499    oversight requirement functional architecture ...
500    edward enterpriseone developer please read sol...
501    develop proprietary service match merge qualit...
502    based austin revolutionary cognitive computing...
503    position important company responsible success...
504    position important company responsible success...
505    applicant need demonstrate deep knowledge proc...
506    position important company responsible success...
507    leading team accounting professional analyst p...
508    leading organization within finance spacewhy h...
509    database architect please read intelligence co...
510    located beautiful palo alto looking lead java ...
511    working tight knit team adding newest platform...
512    hand technical leader hand leadership actively...
513    help drive architecture implementation new fea...
514    leader energetically promote market develop op...
515    leader energetically promote market develop op...
516    leader energetically promote market develop op...
517    differentiate position product solution creati...
518    identify prospect sell market space schedule m...
Name: re, dtype: object

text_list = [i.split() for i in text['re']]
len(text_list)

519

stop_list = ['experience', 'data']
for i in range(len(text_list)):
    for word in list(text_list[i]):  # iterating on a copy since removing will mess things up
        if word in stop_list:
            text_list[i].remove(word)

# Importing Gensim
import gensim
from gensim import corpora

# Creating the term dictionary of our courpus, where every unique term is assigned an index.
dictionary = corpora.Dictionary(text_list)
dictionary.save('dictionary.dict')
print (dictionary)

Dictionary(4878 unique tokens: ['able', 'added', 'analytical', 'applied', 'apply']...)

# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.
doc_term_matrix = [dictionary.doc2bow(doc) for doc in text_list]
corpora.MmCorpus.serialize('corpus.mm', doc_term_matrix)

print (len(doc_term_matrix))
print (doc_term_matrix[100])

519
[(4, 1), (11, 1), (22, 1), (24, 1), (33, 1), (34, 3), (48, 1), (58, 1), (62, 1), (73, 1), (79, 1), (82, 1), (83, 1), (87, 1), (97, 1), (105, 1), (106, 2), (108, 1), (119, 1), (121, 1), (127, 1), (140, 2), (150, 1), (159, 1), (166, 1), (168, 1), (170, 1), (173, 1), (234, 1), (243, 1), (318, 1), (330, 1), (364, 1), (392, 1), (542, 1), (604, 1), (700, 1), (702, 1), (703, 1), (705, 1), (718, 1), (722, 1), (727, 1), (728, 1), (1230, 1), (1264, 1), (1302, 1), (1305, 1), (1323, 1), (1783, 4), (1785, 1), (1786, 1), (1787, 1), (1788, 2)]

from time import time
import logging

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO,
                   filename='running.log',filemode='w')

start = time()
# Creating the object for LDA model using gensim library
Lda = gensim.models.ldamodel.LdaModel

# Running and Trainign LDA model on the document term matrix.
ldamodel = Lda(doc_term_matrix, num_topics=10, id2word = dictionary, passes=50)
print ('used: {:.2f}s'.format(time()-start))

used: 24.50s

for i in ldamodel.print_topics(): 
    for j in i: print (j)

0
0.011*"product" + 0.010*"need" + 0.009*"technology" + 0.009*"knowledge" + 0.009*"customer" + 0.008*"team" + 0.007*"plus" + 0.007*"year" + 0.007*"reporting" + 0.007*"market"
1
0.010*"knowledge" + 0.010*"application" + 0.009*"need" + 0.009*"year" + 0.009*"tool" + 0.008*"user" + 0.008*"technology" + 0.008*"working" + 0.007*"skill" + 0.007*"quality"
2
0.014*"work" + 0.013*"need" + 0.012*"please" + 0.011*"year" + 0.011*"java" + 0.008*"engineer" + 0.008*"company" + 0.008*"python" + 0.008*"solution" + 0.008*"apply"
3
0.010*"tool" + 0.009*"work" + 0.009*"knowledge" + 0.009*"need" + 0.007*"team" + 0.007*"network" + 0.007*"python" + 0.007*"strong" + 0.006*"support" + 0.006*"using"
4
0.023*"learning" + 0.021*"machine" + 0.013*"need" + 0.011*"team" + 0.010*"year" + 0.008*"knowledge" + 0.008*"python" + 0.007*"statistical" + 0.007*"algorithm" + 0.007*"working"
5
0.015*"team" + 0.012*"need" + 0.010*"work" + 0.010*"application" + 0.009*"year" + 0.009*"solution" + 0.009*"technology" + 0.008*"develop" + 0.008*"product" + 0.007*"project"
6
0.024*"knowledge" + 0.014*"network" + 0.011*"java" + 0.010*"highly" + 0.010*"internals" + 0.010*"distributed" + 0.008*"environment" + 0.008*"working" + 0.007*"skill" + 0.007*"time"
7
0.014*"year" + 0.012*"scientist" + 0.012*"passion" + 0.011*"project" + 0.010*"new" + 0.009*"teaching" + 0.009*"cloud" + 0.008*"product" + 0.008*"quarter" + 0.008*"team"
8
0.016*"work" + 0.014*"sql" + 0.011*"year" + 0.010*"team" + 0.010*"performance" + 0.009*"need" + 0.009*"knowledge" + 0.009*"position" + 0.009*"develop" + 0.008*"working"
9
0.017*"product" + 0.012*"infrastructure" + 0.011*"need" + 0.007*"platform" + 0.007*"cloud" + 0.006*"building" + 0.006*"team" + 0.006*"like" + 0.006*"technology" + 0.006*"related"

ldamodel.save('topic.model')

import pyLDAvis.gensim
import gensim
pyLDAvis.enable_notebook()

d = gensim.corpora.Dictionary.load('dictionary.dict')
c = gensim.corpora.MmCorpus('corpus.mm')
lda = gensim.models.LdaModel.load('topic.model')

data = pyLDAvis.gensim.prepare(lda, c, d)
data

pyLDAvis.save_html(data,'vis.html')

%%time
import gensim
import warnings
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
lda = gensim.models.LdaModel.load('topic.model')

CPU times: user 6.96 ms, sys: 3.4 ms, total: 10.4 ms
Wall time: 8.39 ms

fiz=plt.figure(figsize=(15,30))
for i in range(10):
    df=pd.DataFrame(lda.show_topic(i), columns=['term','prob']).set_index('term')
    plt.subplot(5,2,i+1)
    plt.title('topic '+str(i+1))
    sns.barplot(x='prob', y=df.index, data=df, label='Cities', palette='Reds_d')
    plt.xlabel('probability')
    

plt.show()

Description Word Frequency¶

word2count = {}
for data in responsibilities:
    words = nltk.word_tokenize(data)
    for word in words:
        if word not in word2count.keys():
            word2count[word] = 1
        else:
            word2count[word] += 1

#sort words 
from collections import OrderedDict
sorted_word = OrderedDict(sorted(word2count.items(), key=lambda kv: kv[1], reverse=True))
sorted_word

OrderedDict([('need', 605),
             ('team', 536),
             ('year', 515),
             ('knowledge', 482),
             ('work', 481),
             ('product', 349),
             ('working', 344),
             ('technology', 332),
             ('application', 319),
             ('learning', 312),
             ('solution', 306),
             ('tool', 305),
             ('machine', 273),
             ('python', 273),
             ('skill', 269),
             ('strong', 266),
             ('position', 263),
             ('project', 261),
             ('engineer', 241),
             ('develop', 235),
             ('environment', 234),
             ('new', 234),
             ('sql', 233),
             ('service', 230),
             ('platform', 229),
             ('java', 224),
             ('big', 214),
             ('degree', 213),
             ('etc', 213),
             ('please', 207),
             ('cloud', 205),
             ('database', 195),
             ('building', 192),
             ('computer', 190),
             ('plus', 187),
             ('technical', 184),
             ('process', 181),
             ('using', 180),
             ('related', 180),
             ('performance', 172),
             ('ability', 167),
             ('build', 167),
             ('understanding', 166),
             ('customer', 165),
             ('apply', 164),
             ('architecture', 163),
             ('support', 162),
             ('quality', 158),
             ('company', 156),
             ('code', 156),
             ('developing', 155),
             ('high', 151),
             ('including', 150),
             ('user', 147),
             ('problem', 146),
             ('based', 143),
             ('large', 142),
             ('web', 138),
             ('aws', 136),
             ('requirement', 133),
             ('programming', 131),
             ('senior', 129),
             ('language', 127),
             ('spark', 127),
             ('hadoop', 127),
             ('developer', 127),
             ('today', 126),
             ('industry', 126),
             ('field', 125),
             ('distributed', 124),
             ('algorithm', 123),
             ('least', 123),
             ('network', 122),
             ('complex', 122),
             ('end', 122),
             ('processing', 121),
             ('client', 121),
             ('infrastructure', 115),
             ('framework', 115),
             ('integration', 115),
             ('reporting', 114),
             ('time', 114),
             ('scientist', 113),
             ('highly', 113),
             ('bachelor', 112),
             ('practice', 107),
             ('well', 106),
             ('use', 106),
             ('looking', 106),
             ('lead', 103),
             ('best', 103),
             ('create', 102),
             ('required', 102),
             ('linux', 101),
             ('scale', 98),
             ('source', 97),
             ('provide', 97),
             ('intelligence', 96),
             ('role', 95),
             ('across', 94),
             ('like', 94),
             ('feature', 93),
             ('scalable', 92),
             ('information', 92),
             ('following', 91),
             ('report', 91),
             ('multiple', 90),
             ('plan', 88),
             ('help', 87),
             ('top', 86),
             ('opportunity', 85),
             ('sale', 85),
             ('must', 85),
             ('expertise', 85),
             ('maintain', 85),
             ('one', 84),
             ('javascript', 84),
             ('financial', 83),
             ('stack', 83),
             ('etl', 81),
             ('pipeline', 81),
             ('communication', 81),
             ('advanced', 81),
             ('implement', 80),
             ('hand', 80),
             ('expert', 80),
             ('set', 79),
             ('level', 79),
             ('writing', 78),
             ('statistical', 77),
             ('creating', 77),
             ('area', 77),
             ('production', 76),
             ('implementation', 76),
             ('agile', 74),
             ('issue', 74),
             ('scala', 73),
             ('deep', 73),
             ('automated', 73),
             ('operation', 73),
             ('marketing', 72),
             ('master', 72),
             ('enterprise', 71),
             ('within', 70),
             ('preferred', 70),
             ('core', 70),
             ('professional', 70),
             ('identify', 69),
             ('manager', 69),
             ('market', 69),
             ('analyst', 69),
             ('designing', 68),
             ('positionat', 68),
             ('responsible', 67),
             ('architect', 67),
             ('drive', 66),
             ('ensure', 66),
             ('read', 66),
             ('candidate', 66),
             ('passion', 65),
             ('full', 65),
             ('key', 65),
             ('partner', 65),
             ('improve', 63),
             ('understand', 63),
             ('driven', 63),
             ('real', 63),
             ('solid', 63),
             ('various', 63),
             ('control', 62),
             ('relational', 62),
             ('multi', 62),
             ('equivalent', 61),
             ('research', 61),
             ('storage', 60),
             ('part', 60),
             ('automation', 60),
             ('analyze', 59),
             ('solving', 59),
             ('pattern', 59),
             ('standard', 59),
             ('leading', 59),
             ('able', 58),
             ('implementing', 58),
             ('strategy', 57),
             ('fast', 56),
             ('learn', 55),
             ('make', 54),
             ('technique', 54),
             ('relevant', 54),
             ('scripting', 54),
             ('include', 54),
             ('nosql', 54),
             ('closely', 54),
             ('excellent', 53),
             ('visualization', 53),
             ('review', 53),
             ('computing', 52),
             ('tableau', 52),
             ('functional', 51),
             ('leadership', 51),
             ('organization', 51),
             ('familiarity', 51),
             ('configuration', 51),
             ('background', 50),
             ('around', 50),
             ('oriented', 50),
             ('result', 50),
             ('existing', 50),
             ('api', 50),
             ('deployment', 50),
             ('training', 49),
             ('open', 48),
             ('internal', 48),
             ('growing', 48),
             ('also', 48),
             ('proficient', 48),
             ('phd', 47),
             ('improvement', 47),
             ('perform', 47),
             ('cross', 47),
             ('react', 47),
             ('good', 46),
             ('interface', 46),
             ('member', 46),
             ('continuous', 46),
             ('delivery', 46),
             ('mining', 45),
             ('document', 45),
             ('decision', 45),
             ('managing', 45),
             ('methodology', 45),
             ('supporting', 45),
             ('program', 45),
             ('object', 45),
             ('analytical', 44),
             ('excel', 44),
             ('dynamic', 44),
             ('ruby', 44),
             ('oracle', 44),
             ('angular', 44),
             ('coding', 43),
             ('net', 43),
             ('meet', 43),
             ('query', 43),
             ('unit', 43),
             ('statistic', 42),
             ('small', 42),
             ('similar', 42),
             ('kafka', 42),
             ('microsoft', 42),
             ('monitor', 41),
             ('structure', 41),
             ('want', 41),
             ('relationship', 41),
             ('day', 41),
             ('mobile', 41),
             ('participate', 41),
             ('html', 41),
             ('monitoring', 40),
             ('written', 40),
             ('responsibility', 40),
             ('collaborate', 40),
             ('join', 40),
             ('window', 40),
             ('salesforce', 40),
             ('warehouse', 39),
             ('stakeholder', 39),
             ('resource', 39),
             ('reason', 39),
             ('center', 39),
             ('case', 39),
             ('paced', 39),
             ('documentation', 39),
             ('quantitative', 38),
             ('directly', 38),
             ('positionmust', 38),
             ('networking', 38),
             ('capability', 37),
             ('way', 37),
             ('world', 37),
             ('healthcare', 37),
             ('deliver', 37),
             ('edge', 37),
             ('vision', 37),
             ('deploying', 37),
             ('operating', 37),
             ('component', 37),
             ('innovative', 36),
             ('providing', 36),
             ('assist', 36),
             ('write', 36),
             ('analyzing', 36),
             ('consulting', 36),
             ('ha', 36),
             ('cycle', 36),
             ('power', 36),
             ('com', 36),
             ('access', 36),
             ('administration', 36),
             ('cisco', 36),
             ('certification', 36),
             ('hardware', 36),
             ('rpa', 36),
             ('positionmore', 35),
             ('library', 35),
             ('tech', 35),
             ('minimum', 35),
             ('method', 35),
             ('leader', 35),
             ('positionrequirements', 35),
             ('maintaining', 35),
             ('predictive', 34),
             ('mysql', 34),
             ('self', 34),
             ('career', 34),
             ('cassandra', 34),
             ('life', 34),
             ('effort', 34),
             ('growth', 34),
             ('great', 34),
             ('office', 34),
             ('image', 34),
             ('maintenance', 34),
             ('backend', 34),
             ('procedure', 34),
             ('rest', 34),
             ('front', 34),
             ('manage', 34),
             ('spring', 34),
             ('insight', 33),
             ('used', 33),
             ('ideal', 33),
             ('focused', 33),
             ('solve', 33),
             ('teaching', 33),
             ('employee', 33),
             ('stream', 33),
             ('git', 33),
             ('effectively', 33),
             ('apis', 33),
             ('azure', 33),
             ('would', 32),
             ('optimization', 32),
             ('back', 32),
             ('proven', 32),
             ('event', 32),
             ('google', 32),
             ('daily', 32),
             ('concept', 32),
             ('applied', 31),
             ('startup', 31),
             ('different', 31),
             ('passionate', 31),
             ('goal', 31),
             ('space', 31),
             ('ssis', 31),
             ('troubleshooting', 31),
             ('task', 31),
             ('communicate', 30),
             ('function', 30),
             ('external', 30),
             ('right', 30),
             ('basic', 30),
             ('love', 30),
             ('approach', 30),
             ('benefit', 30),
             ('manufacturing', 30),
             ('fundamental', 30),
             ('cs', 30),
             ('availability', 29),
             ('current', 29),
             ('electrical', 29),
             ('interest', 29),
             ('amount', 29),
             ('san', 29),
             ('hard', 29),
             ('next', 29),
             ('unix', 29),
             ('protocol', 29),
             ('individual', 28),
             ('metric', 28),
             ('robust', 28),
             ('consumer', 28),
             ('presentation', 28),
             ('comfortable', 28),
             ('discipline', 28),
             ('domain', 28),
             ('cutting', 28),
             ('variety', 28),
             ('dashboard', 28),
             ('start', 28),
             ('specification', 28),
             ('desired', 28),
             ('bash', 28),
             ('ecosystem', 27),
             ('define', 27),
             ('math', 27),
             ('record', 27),
             ('detail', 27),
             ('evaluate', 27),
             ('optimize', 27),
             ('integrate', 27),
             ('mongodb', 27),
             ('take', 27),
             ('restful', 27),
             ('focus', 26),
             ('line', 26),
             ('objective', 26),
             ('strategic', 26),
             ('future', 26),
             ('innovation', 26),
             ('recommendation', 26),
             ('others', 26),
             ('nlp', 26),
             ('hive', 26),
             ('order', 26),
             ('generation', 26),
             ('lifecycle', 26),
             ('operational', 26),
             ('proficiency', 26),
             ('active', 26),
             ('critical', 26),
             ('department', 26),
             ('nice', 26),
             ('internals', 26),
             ('facing', 26),
             ('commerce', 25),
             ('successful', 25),
             ('redshift', 25),
             ('global', 25),
             ('base', 25),
             ('segment', 25),
             ('course', 25),
             ('quarter', 25),
             ('search', 25),
             ('basis', 25),
             ('finance', 25),
             ('artificial', 25),
             ('docker', 25),
             ('travel', 25),
             ('effective', 25),
             ('activity', 25),
             ('change', 25),
             ('general', 25),
             ('direction', 25),
             ('efficiency', 25),
             ('equipment', 25),
             ('risk', 24),
             ('determine', 24),
             ('reliability', 24),
             ('trend', 24),
             ('mentor', 24),
             ('success', 24),
             ('package', 24),
             ('school', 24),
             ('offering', 24),
             ('ideally', 24),
             ('bonus', 24),
             ('subject', 24),
             ('includes', 24),
             ('talented', 24),
             ('point', 24),
             ('planning', 24),
             ('mentoring', 24),
             ('firewall', 24),
             ('mathematics', 23),
             ('flow', 23),
             ('clean', 23),
             ('range', 23),
             ('pig', 23),
             ('helping', 23),
             ('act', 23),
             ('matter', 23),
             ('contribute', 23),
             ('performing', 23),
             ('provides', 23),
             ('needed', 23),
             ('amazon', 23),
             ('postgresql', 23),
             ('cost', 23),
             ('handle', 23),
             ('sdlc', 23),
             ('json', 23),
             ('saas', 23),
             ('extensive', 23),
             ('complete', 23),
             ('job', 23),
             ('previous', 22),
             ('competitive', 22),
             ('latest', 22),
             ('huge', 22),
             ('specific', 22),
             ('appropriate', 22),
             ('class', 22),
             ('play', 22),
             ('meeting', 22),
             ('hbase', 22),
             ('delivering', 22),
             ('initiative', 22),
             ('familiar', 22),
             ('modern', 22),
             ('value', 22),
             ('shell', 22),
             ('tuning', 22),
             ('router', 22),
             ('ssrs', 22),
             ('duty', 22),
             ('side', 22),
             ('overall', 21),
             ('track', 21),
             ('structured', 21),
             ('impact', 21),
             ('idea', 21),
             ('question', 21),
             ('enhance', 21),
             ('interested', 21),
             ('streaming', 21),
             ('experienced', 21),
             ('scaling', 21),
             ('challenge', 21),
             ('load', 21),
             ('wan', 21),
             ('node', 21),
             ('tensorflow', 20),
             ('built', 20),
             ('people', 20),
             ('series', 20),
             ('medium', 20),
             ('non', 20),
             ('scalability', 20),
             ('fun', 20),
             ('posse', 20),
             ('natural', 20),
             ('grow', 20),
             ('group', 20),
             ('smart', 20),
             ('aspect', 20),
             ('interact', 20),
             ('may', 20),
             ('hire', 20),
             ('script', 20),
             ('clearance', 20),
             ('device', 20),
             ('jquery', 20),
             ('assurance', 20),
             ('extraction', 19),
             ('gather', 19),
             ('emr', 19),
             ('involved', 19),
             ('prototype', 19),
             ('creative', 19),
             ('apache', 19),
             ('matlab', 19),
             ('collaborative', 19),
             ('table', 19),
             ('established', 19),
             ('rapidly', 19),
             ('deploy', 19),
             ('quickly', 19),
             ('account', 19),
             ('massive', 19),
             ('currently', 19),
             ('store', 19),
             ('layer', 19),
             ('establish', 19),
             ('release', 19),
             ('priority', 19),
             ('tcp', 19),
             ('io', 19),
             ('depth', 18),
             ('primary', 18),
             ('someone', 18),
             ('resume', 18),
             ('willingness', 18),
             ('diverse', 18),
             ('validation', 18),
             ('ecommerce', 18),
             ('material', 18),
             ('revenue', 18),
             ('budget', 18),
             ('public', 18),
             ('digital', 18),
             ('debugging', 18),
             ('serve', 18),
             ('storm', 18),
             ('staff', 18),
             ('trading', 18),
             ('version', 18),
             ('request', 18),
             ('human', 18),
             ('integrity', 18),
             ('junior', 18),
             ('even', 18),
             ('advantageous', 18),
             ('hybris', 18),
             ('suite', 17),
             ('verbal', 17),
             ('improving', 17),
             ('conduct', 17),
             ('person', 17),
             ('get', 17),
             ('state', 17),
             ('three', 17),
             ('retrieval', 17),
             ('push', 17),
             ('optimizing', 17),
             ('kubernetes', 17),
             ('provider', 17),
             ('million', 17),
             ('bscs', 17),
             ('elasticsearch', 17),
             ('date', 17),
             ('cybercoders', 17),
             ('site', 17),
             ('advertising', 17),
             ('location', 17),
             ('necessary', 17),
             ('available', 17),
             ('microservices', 17),
             ('xml', 17),
             ('chain', 17),
             ('switch', 17),
             ('come', 17),
             ('timely', 17),
             ('interpersonal', 17),
             ('post', 17),
             ('mvc', 17),
             ('file', 17),
             ('accounting', 17),
             ('assessment', 16),
             ('custom', 16),
             ('present', 16),
             ('towards', 16),
             ('common', 16),
             ('execution', 16),
             ('regression', 16),
             ('le', 16),
             ('potential', 16),
             ('discussion', 16),
             ('challenging', 16),
             ('mapreduce', 16),
             ('functionality', 16),
             ('volume', 16),
             ('drawing', 16),
             ('keep', 16),
             ('detailed', 16),
             ('bring', 16),
             ('demonstrated', 16),
             ('exciting', 16),
             ('behavior', 16),
             ('prior', 16),
             ('find', 16),
             ('handling', 16),
             ('maintainable', 16),
             ('party', 16),
             ('desire', 16),
             ('container', 16),
             ('tdd', 16),
             ('devops', 16),
             ('along', 16),
             ('signal', 16),
             ('oop', 16),
             ('practical', 16),
             ('hibernate', 16),
             ('wireless', 16),
             ('many', 15),
             ('prepare', 15),
             ('collaboration', 15),
             ('important', 15),
             ('validate', 15),
             ('wide', 15),
             ('execute', 15),
             ('two', 15),
             ('enhancing', 15),
             ('demand', 15),
             ('apps', 15),
             ('vendor', 15),
             ('form', 15),
             ('evaluation', 15),
             ('tier', 15),
             ('major', 15),
             ('php', 15),
             ('either', 15),
             ('warehousing', 15),
             ('scope', 15),
             ('parallel', 15),
             ('orchestration', 15),
             ('troubleshoot', 15),
             ('frontend', 15),
             ('interaction', 15),
             ('distribution', 15),
             ('micro', 15),
             ('limited', 15),
             ('competent', 15),
             ('concurrency', 15),
             ('apex', 15),
             ('qualification', 15),
             ('preferably', 15),
             ('schedule', 15),
             ('incident', 15),
             ('increase', 14),
             ('coordinate', 14),
             ('neural', 14),
             ('transformation', 14),
             ('finding', 14),
             ('know', 14),
             ('analytic', 14),
             ('willing', 14),
             ('memory', 14),
             ('unstructured', 14),
             ('commercial', 14),
             ('presenting', 14),
             ('independently', 14),
             ('raw', 14),
             ('expand', 14),
             ('care', 14),
             ('student', 14),
             ('month', 14),
             ('incomplete', 14),
             ('salary', 14),
             ('medical', 14),
             ('principle', 14),
             ('blockchain', 14),
             ('motivated', 14),
             ('develops', 14),
             ('ensuring', 14),
             ('produce', 14),
             ('integrating', 14),
             ('mission', 14),
             ('scrum', 14),
             ('direct', 14),
             ('resolve', 14),
             ('bug', 14),
             ('email', 14),
             ('transfer', 14),
             ('browser', 14),
             ('ccnp', 14),
             ('train', 14),
             ('architectural', 14),
             ('messaging', 14),
             ('personnel', 14),
             ('engine', 13),
             ('detection', 13),
             ('stay', 13),
             ('address', 13),
             ('number', 13),
             ('proprietary', 13),
             ('local', 13),
             ('fit', 13),
             ('engaging', 13),
             ('recommend', 13),
             ('forecast', 13),
             ('thrive', 13),
             ('interview', 13),
             ('online', 13),
             ('culture', 13),
             ('generate', 13),
             ('outside', 13),
             ('designer', 13),
             ('offer', 13),
             ('embedded', 13),
             ('hdfs', 13),
             ('identifying', 13),
             ('long', 13),
             ('response', 13),
             ('peer', 13),
             ('funded', 13),
             ('send', 13),
             ('found', 13),
             ('app', 13),
             ('higher', 13),
             ('secure', 13),
             ('resolution', 13),
             ('update', 13),
             ('coverage', 13),
             ('entity', 13),
             ('maven', 13),
             ('proactively', 13),
             ('contract', 13),
             ('brand', 13),
             ('accurate', 13),
             ('jira', 13),
             ('entire', 13),
             ('http', 13),
             ('android', 13),
             ('upgrade', 13),
             ('content', 13),
             ('utilizing', 12),
             ('closed', 12),
             ('leverage', 12),
             ('quick', 12),
             ('hour', 12),
             ('guidance', 12),
             ('screening', 12),
             ('option', 12),
             ('every', 12),
             ('postgres', 12),
             ('contributing', 12),
             ('extract', 12),
             ('term', 12),
             ('iot', 12),
             ('actionable', 12),
             ('sci', 12),
             ('fully', 12),
             ('emerging', 12),
             ('optimal', 12),
             ('massively', 12),
             ('low', 12),
             ('voice', 12),
             ('datacenter', 12),
             ('virtualization', 12),
             ('soap', 12),
             ('tracking', 12),
             ('name', 12),
             ('positionrequired', 12),
             ('additional', 12),
             ('enhancement', 12),
             ('status', 12),
             ('regular', 12),
             ('organizational', 12),
             ('collaboratively', 12),
             ('routing', 12),
             ('robotic', 12),
             ('engage', 12),
             ('robotics', 12),
             ('run', 12),
             ('logic', 12),
             ('visual', 12),
             ('policy', 12),
             ('acceptance', 12),
             ('monthly', 12),
             ('enterpriseone', 12),
             ('payment', 11),
             ('leveraging', 11),
             ('accuracy', 11),
             ('launch', 11),
             ('diagram', 11),
             ('home', 11),
             ('acquisition', 11),
             ('capacity', 11),
             ('health', 11),
             ('add', 11),
             ('extremely', 11),
             ('intelligent', 11),
             ('facilitate', 11),
             ('interpret', 11),
             ('history', 11),
             ('datasets', 11),
             ('spend', 11),
             ('speaking', 11),
             ('whatever', 11),
             ('answer', 11),
             ('enjoy', 11),
             ('interesting', 11),
             ('stability', 11),
             ('influence', 11),
             ('roadmap', 11),
             ('third', 11),
             ('prototyping', 11),
             ('portfolio', 11),
             ('funding', 11),
             ('driving', 11),
             ('combination', 11),
             ('efficient', 11),
             ('another', 11),
             ('move', 11),
             ('root', 11),
             ('cause', 11),
             ('collection', 11),
             ('verification', 11),
             ('estate', 11),
             ('automate', 11),
             ('yr', 11),
             ('chef', 11),
             ('ownership', 11),
             ('broad', 11),
             ('exposure', 11),
             ('jvm', 11),
             ('ground', 11),
             ('redis', 11),
             ('reduce', 11),
             ('input', 11),
             ('successfully', 11),
             ('word', 11),
             ('constantly', 11),
             ('asset', 11),
             ('clear', 11),
             ('positionexperience', 11),
             ('threaded', 11),
             ('sdn', 11),
             ('controller', 11),
             ('imaging', 11),
             ('circuit', 11),
             ('experiment', 10),
             ('effectiveness', 10),
             ('scikit', 10),
             ('changing', 10),
             ('considered', 10),
             ('predict', 10),
             ('collect', 10),
             ('kera', 10),
             ('player', 10),
             ('explore', 10),
             ('collaborating', 10),
             ('sa', 10),
             ('cluster', 10),
             ('beyond', 10),
             ('call', 10),
             ('conference', 10),
             ('essential', 10),
             ('jose', 10),
             ('thousand', 10),
             ('guide', 10),
             ('setting', 10),
             ('plusso', 10),
             ('integral', 10),
             ('division', 10),
             ('expected', 10),
             ('audience', 10),
             ('communicating', 10),
             ('raleigh', 10),
             ('gap', 10),
             ('deal', 10),
             ('strongly', 10),
             ('metadata', 10),
             ('kinesis', 10),
             ('zookeeper', 10),
             ('outstanding', 10),
             ('puppet', 10),
             ('reliable', 10),
             ('mdm', 10),
             ('compensation', 10),
             ('recently', 10),
             ('mentality', 10),
             ('na', 10),
             ('ansible', 10),
             ('hoc', 10),
             ('associated', 10),
             ('github', 10),
             ('stable', 10),
             ('assigned', 10),
             ('rapid', 10),
             ('ccie', 10),
             ('prevention', 10),
             ('speed', 10),
             ('jenkins', 10),
             ('accomplish', 10),
             ('dns', 10),
             ('premise', 10),
             ('compliance', 10),
             ('backbone', 10),
             ('follow', 10),
             ('threading', 10),
             ('iso', 10),
             ('asp', 10),
             ('lan', 10),
             ('setup', 10),
             ('deadline', 10),
             ('module', 10),
             ('video', 10),
             ('construction', 10),
             ('oral', 10),
             ('director', 10),
             ('edward', 10),
             ('jde', 10),
             ('scientific', 9),
             ('early', 9),
             ('outcome', 9),
             ('feedback', 9),
             ('round', 9),
             ('random', 9),
             ('forward', 9),
             ('gain', 9),
             ('aggregation', 9),
             ('attention', 9),
             ('scenario', 9),
             ('theory', 9),
             ('without', 9),
             ('satisfaction', 9),
             ('reading', 9),
             ('log', 9),
             ('college', 9),
             ('simple', 9),
             ('incredible', 9),
             ('instructor', 9),
             ('assisting', 9),
             ('hiring', 9),
             ('unlimited', 9),
             ('pto', 9),
             ('throughout', 9),
             ('heterogeneous', 9),
             ('index', 9),
             ('text', 9),
             ('panda', 9),
             ('corporate', 9),
             ('creation', 9),
             ('physic', 9),
             ('competency', 9),
             ('unique', 9),
             ...])

#choose most frequent ones
freq_words = heapq.nlargest(1000,word2count,key = word2count.get)
freq_words

['need',
 'team',
 'year',
 'knowledge',
 'work',
 'product',
 'working',
 'technology',
 'application',
 'learning',
 'solution',
 'tool',
 'machine',
 'python',
 'skill',
 'strong',
 'position',
 'project',
 'engineer',
 'develop',
 'environment',
 'new',
 'sql',
 'service',
 'platform',
 'java',
 'big',
 'degree',
 'etc',
 'please',
 'cloud',
 'database',
 'building',
 'computer',
 'plus',
 'technical',
 'process',
 'using',
 'related',
 'performance',
 'ability',
 'build',
 'understanding',
 'customer',
 'apply',
 'architecture',
 'support',
 'quality',
 'company',
 'code',
 'developing',
 'high',
 'including',
 'user',
 'problem',
 'based',
 'large',
 'web',
 'aws',
 'requirement',
 'programming',
 'senior',
 'language',
 'spark',
 'hadoop',
 'developer',
 'today',
 'industry',
 'field',
 'distributed',
 'algorithm',
 'least',
 'network',
 'complex',
 'end',
 'processing',
 'client',
 'infrastructure',
 'framework',
 'integration',
 'reporting',
 'time',
 'scientist',
 'highly',
 'bachelor',
 'practice',
 'well',
 'use',
 'looking',
 'lead',
 'best',
 'create',
 'required',
 'linux',
 'scale',
 'source',
 'provide',
 'intelligence',
 'role',
 'across',
 'like',
 'feature',
 'scalable',
 'information',
 'following',
 'report',
 'multiple',
 'plan',
 'help',
 'top',
 'opportunity',
 'sale',
 'must',
 'expertise',
 'maintain',
 'one',
 'javascript',
 'financial',
 'stack',
 'etl',
 'pipeline',
 'communication',
 'advanced',
 'implement',
 'hand',
 'expert',
 'set',
 'level',
 'writing',
 'statistical',
 'creating',
 'area',
 'production',
 'implementation',
 'agile',
 'issue',
 'scala',
 'deep',
 'automated',
 'operation',
 'marketing',
 'master',
 'enterprise',
 'within',
 'preferred',
 'core',
 'professional',
 'identify',
 'manager',
 'market',
 'analyst',
 'designing',
 'positionat',
 'responsible',
 'architect',
 'drive',
 'ensure',
 'read',
 'candidate',
 'passion',
 'full',
 'key',
 'partner',
 'improve',
 'understand',
 'driven',
 'real',
 'solid',
 'various',
 'control',
 'relational',
 'multi',
 'equivalent',
 'research',
 'storage',
 'part',
 'automation',
 'analyze',
 'solving',
 'pattern',
 'standard',
 'leading',
 'able',
 'implementing',
 'strategy',
 'fast',
 'learn',
 'make',
 'technique',
 'relevant',
 'scripting',
 'include',
 'nosql',
 'closely',
 'excellent',
 'visualization',
 'review',
 'computing',
 'tableau',
 'functional',
 'leadership',
 'organization',
 'familiarity',
 'configuration',
 'background',
 'around',
 'oriented',
 'result',
 'existing',
 'api',
 'deployment',
 'training',
 'open',
 'internal',
 'growing',
 'also',
 'proficient',
 'phd',
 'improvement',
 'perform',
 'cross',
 'react',
 'good',
 'interface',
 'member',
 'continuous',
 'delivery',
 'mining',
 'document',
 'decision',
 'managing',
 'methodology',
 'supporting',
 'program',
 'object',
 'analytical',
 'excel',
 'dynamic',
 'ruby',
 'oracle',
 'angular',
 'coding',
 'net',
 'meet',
 'query',
 'unit',
 'statistic',
 'small',
 'similar',
 'kafka',
 'microsoft',
 'monitor',
 'structure',
 'want',
 'relationship',
 'day',
 'mobile',
 'participate',
 'html',
 'monitoring',
 'written',
 'responsibility',
 'collaborate',
 'join',
 'window',
 'salesforce',
 'warehouse',
 'stakeholder',
 'resource',
 'reason',
 'center',
 'case',
 'paced',
 'documentation',
 'quantitative',
 'directly',
 'positionmust',
 'networking',
 'capability',
 'way',
 'world',
 'healthcare',
 'deliver',
 'edge',
 'vision',
 'deploying',
 'operating',
 'component',
 'innovative',
 'providing',
 'assist',
 'write',
 'analyzing',
 'consulting',
 'ha',
 'cycle',
 'power',
 'com',
 'access',
 'administration',
 'cisco',
 'certification',
 'hardware',
 'rpa',
 'positionmore',
 'library',
 'tech',
 'minimum',
 'method',
 'leader',
 'positionrequirements',
 'maintaining',
 'predictive',
 'mysql',
 'self',
 'career',
 'cassandra',
 'life',
 'effort',
 'growth',
 'great',
 'office',
 'image',
 'maintenance',
 'backend',
 'procedure',
 'rest',
 'front',
 'manage',
 'spring',
 'insight',
 'used',
 'ideal',
 'focused',
 'solve',
 'teaching',
 'employee',
 'stream',
 'git',
 'effectively',
 'apis',
 'azure',
 'would',
 'optimization',
 'back',
 'proven',
 'event',
 'google',
 'daily',
 'concept',
 'applied',
 'startup',
 'different',
 'passionate',
 'goal',
 'space',
 'ssis',
 'troubleshooting',
 'task',
 'communicate',
 'function',
 'external',
 'right',
 'basic',
 'love',
 'approach',
 'benefit',
 'manufacturing',
 'fundamental',
 'cs',
 'availability',
 'current',
 'electrical',
 'interest',
 'amount',
 'san',
 'hard',
 'next',
 'unix',
 'protocol',
 'individual',
 'metric',
 'robust',
 'consumer',
 'presentation',
 'comfortable',
 'discipline',
 'domain',
 'cutting',
 'variety',
 'dashboard',
 'start',
 'specification',
 'desired',
 'bash',
 'ecosystem',
 'define',
 'math',
 'record',
 'detail',
 'evaluate',
 'optimize',
 'integrate',
 'mongodb',
 'take',
 'restful',
 'focus',
 'line',
 'objective',
 'strategic',
 'future',
 'innovation',
 'recommendation',
 'others',
 'nlp',
 'hive',
 'order',
 'generation',
 'lifecycle',
 'operational',
 'proficiency',
 'active',
 'critical',
 'department',
 'nice',
 'internals',
 'facing',
 'commerce',
 'successful',
 'redshift',
 'global',
 'base',
 'segment',
 'course',
 'quarter',
 'search',
 'basis',
 'finance',
 'artificial',
 'docker',
 'travel',
 'effective',
 'activity',
 'change',
 'general',
 'direction',
 'efficiency',
 'equipment',
 'risk',
 'determine',
 'reliability',
 'trend',
 'mentor',
 'success',
 'package',
 'school',
 'offering',
 'ideally',
 'bonus',
 'subject',
 'includes',
 'talented',
 'point',
 'planning',
 'mentoring',
 'firewall',
 'mathematics',
 'flow',
 'clean',
 'range',
 'pig',
 'helping',
 'act',
 'matter',
 'contribute',
 'performing',
 'provides',
 'needed',
 'amazon',
 'postgresql',
 'cost',
 'handle',
 'sdlc',
 'json',
 'saas',
 'extensive',
 'complete',
 'job',
 'previous',
 'competitive',
 'latest',
 'huge',
 'specific',
 'appropriate',
 'class',
 'play',
 'meeting',
 'hbase',
 'delivering',
 'initiative',
 'familiar',
 'modern',
 'value',
 'shell',
 'tuning',
 'router',
 'ssrs',
 'duty',
 'side',
 'overall',
 'track',
 'structured',
 'impact',
 'idea',
 'question',
 'enhance',
 'interested',
 'streaming',
 'experienced',
 'scaling',
 'challenge',
 'load',
 'wan',
 'node',
 'tensorflow',
 'built',
 'people',
 'series',
 'medium',
 'non',
 'scalability',
 'fun',
 'posse',
 'natural',
 'grow',
 'group',
 'smart',
 'aspect',
 'interact',
 'may',
 'hire',
 'script',
 'clearance',
 'device',
 'jquery',
 'assurance',
 'extraction',
 'gather',
 'emr',
 'involved',
 'prototype',
 'creative',
 'apache',
 'matlab',
 'collaborative',
 'table',
 'established',
 'rapidly',
 'deploy',
 'quickly',
 'account',
 'massive',
 'currently',
 'store',
 'layer',
 'establish',
 'release',
 'priority',
 'tcp',
 'io',
 'depth',
 'primary',
 'someone',
 'resume',
 'willingness',
 'diverse',
 'validation',
 'ecommerce',
 'material',
 'revenue',
 'budget',
 'public',
 'digital',
 'debugging',
 'serve',
 'storm',
 'staff',
 'trading',
 'version',
 'request',
 'human',
 'integrity',
 'junior',
 'even',
 'advantageous',
 'hybris',
 'suite',
 'verbal',
 'improving',
 'conduct',
 'person',
 'get',
 'state',
 'three',
 'retrieval',
 'push',
 'optimizing',
 'kubernetes',
 'provider',
 'million',
 'bscs',
 'elasticsearch',
 'date',
 'cybercoders',
 'site',
 'advertising',
 'location',
 'necessary',
 'available',
 'microservices',
 'xml',
 'chain',
 'switch',
 'come',
 'timely',
 'interpersonal',
 'post',
 'mvc',
 'file',
 'accounting',
 'assessment',
 'custom',
 'present',
 'towards',
 'common',
 'execution',
 'regression',
 'le',
 'potential',
 'discussion',
 'challenging',
 'mapreduce',
 'functionality',
 'volume',
 'drawing',
 'keep',
 'detailed',
 'bring',
 'demonstrated',
 'exciting',
 'behavior',
 'prior',
 'find',
 'handling',
 'maintainable',
 'party',
 'desire',
 'container',
 'tdd',
 'devops',
 'along',
 'signal',
 'oop',
 'practical',
 'hibernate',
 'wireless',
 'many',
 'prepare',
 'collaboration',
 'important',
 'validate',
 'wide',
 'execute',
 'two',
 'enhancing',
 'demand',
 'apps',
 'vendor',
 'form',
 'evaluation',
 'tier',
 'major',
 'php',
 'either',
 'warehousing',
 'scope',
 'parallel',
 'orchestration',
 'troubleshoot',
 'frontend',
 'interaction',
 'distribution',
 'micro',
 'limited',
 'competent',
 'concurrency',
 'apex',
 'qualification',
 'preferably',
 'schedule',
 'incident',
 'increase',
 'coordinate',
 'neural',
 'transformation',
 'finding',
 'know',
 'analytic',
 'willing',
 'memory',
 'unstructured',
 'commercial',
 'presenting',
 'independently',
 'raw',
 'expand',
 'care',
 'student',
 'month',
 'incomplete',
 'salary',
 'medical',
 'principle',
 'blockchain',
 'motivated',
 'develops',
 'ensuring',
 'produce',
 'integrating',
 'mission',
 'scrum',
 'direct',
 'resolve',
 'bug',
 'email',
 'transfer',
 'browser',
 'ccnp',
 'train',
 'architectural',
 'messaging',
 'personnel',
 'engine',
 'detection',
 'stay',
 'address',
 'number',
 'proprietary',
 'local',
 'fit',
 'engaging',
 'recommend',
 'forecast',
 'thrive',
 'interview',
 'online',
 'culture',
 'generate',
 'outside',
 'designer',
 'offer',
 'embedded',
 'hdfs',
 'identifying',
 'long',
 'response',
 'peer',
 'funded',
 'send',
 'found',
 'app',
 'higher',
 'secure',
 'resolution',
 'update',
 'coverage',
 'entity',
 'maven',
 'proactively',
 'contract',
 'brand',
 'accurate',
 'jira',
 'entire',
 'http',
 'android',
 'upgrade',
 'content',
 'utilizing',
 'closed',
 'leverage',
 'quick',
 'hour',
 'guidance',
 'screening',
 'option',
 'every',
 'postgres',
 'contributing',
 'extract',
 'term',
 'iot',
 'actionable',
 'sci',
 'fully',
 'emerging',
 'optimal',
 'massively',
 'low',
 'voice',
 'datacenter',
 'virtualization',
 'soap',
 'tracking',
 'name',
 'positionrequired',
 'additional',
 'enhancement',
 'status',
 'regular',
 'organizational',
 'collaboratively',
 'routing',
 'robotic',
 'engage',
 'robotics',
 'run',
 'logic',
 'visual',
 'policy',
 'acceptance',
 'monthly',
 'enterpriseone',
 'payment',
 'leveraging',
 'accuracy',
 'launch',
 'diagram',
 'home',
 'acquisition',
 'capacity',
 'health',
 'add',
 'extremely',
 'intelligent',
 'facilitate',
 'interpret',
 'history',
 'datasets',
 'spend',
 'speaking',
 'whatever',
 'answer',
 'enjoy',
 'interesting',
 'stability',
 'influence',
 'roadmap',
 'third',
 'prototyping',
 'portfolio',
 'funding',
 'driving',
 'combination',
 'efficient',
 'another',
 'move',
 'root',
 'cause',
 'collection',
 'verification',
 'estate',
 'automate',
 'yr',
 'chef',
 'ownership',
 'broad',
 'exposure',
 'jvm',
 'ground',
 'redis',
 'reduce',
 'input',
 'successfully',
 'word',
 'constantly',
 'asset',
 'clear',
 'positionexperience',
 'threaded',
 'sdn',
 'controller',
 'imaging',
 'circuit',
 'experiment',
 'effectiveness',
 'scikit',
 'changing',
 'considered',
 'predict',
 'collect',
 'kera',
 'player',
 'explore',
 'collaborating',
 'sa',
 'cluster',
 'beyond',
 'call',
 'conference',
 'essential',
 'jose',
 'thousand',
 'guide',
 'setting',
 'plusso',
 'integral',
 'division',
 'expected',
 'audience',
 'communicating',
 'raleigh',
 'gap',
 'deal',
 'strongly',
 'metadata',
 'kinesis',
 'zookeeper',
 'outstanding',
 'puppet',
 'reliable',
 'mdm',
 'compensation',
 'recently',
 'mentality',
 'na',
 'ansible',
 'hoc',
 'associated',
 'github',
 'stable',
 'assigned',
 'rapid',
 'ccie',
 'prevention',
 'speed',
 'jenkins',
 'accomplish',
 'dns',
 'premise',
 'compliance',
 'backbone',
 'follow',
 'threading',
 'iso',
 'asp',
 'lan',
 'setup',
 'deadline',
 'module',
 'video',
 'construction',
 'oral',
 'director',
 'edward',
 'jde',
 'scientific',
 'early',
 'outcome',
 'feedback',
 'round',
 'random',
 'forward',
 'gain',
 'aggregation',
 'attention',
 'scenario',
 'theory',
 'without',
 'satisfaction',
 'reading',
 'log',
 'college',
 'simple',
 'incredible',
 'instructor',
 'assisting',
 'hiring',
 'unlimited',
 'pto',
 'throughout',
 'heterogeneous',
 'index',
 'text',
 'panda',
 'corporate',
 'creation',
 'physic',
 'competency',
 'unique']

word_frequencies =' '.join([str(i) for i in freq_words])

frequent_words = WordCloud(relative_scaling = 0, stopwords = stop_words, width = 600, height = 300)
frequent_words.generate(word_frequencies)
plt.figure(figsize = (15,7.5))
plt.imshow(frequent_words.recolor(color_func = grey_color_func, random_state = 1), interpolation = "bilinear")
plt.axis("off")
plt.tight_layout(pad = -1)
plt.title("Description - Frequent Words")
plt.show()

Salary Analysis¶

What are you expected to earn in each state?¶

min_state_df = master_data[master_data['Min_Salary'] != 'Unknown'].reset_index(drop = True)
max_state_df = master_data[master_data['Max_Salary'] != 'Unknown'].reset_index(drop = True)

min_state_set = np.unique(min_state_df['States'])
max_state_set = np.unique(max_state_df['States'])

min_state_dict = dict(tuple(min_state_df.groupby('States')))
max_state_dict = dict(tuple(max_state_df.groupby('States')))

min_state_salaries = {}
max_state_salaries = {}
for state in min_state_set:
    avg_min_salary = int(min_state_dict[state]['Min_Salary'].mean())
    min_state_salaries.update({state : avg_min_salary})

for state in max_state_set:
    avg_max_salary = int(max_state_dict[state]['Max_Salary'].mean())
    max_state_salaries.update({state : avg_max_salary})
    
min_state_salaries = OrderedDict(sorted(min_state_salaries.items(), key=lambda t: t[1]))
max_state_salaries = OrderedDict(sorted(max_state_salaries.items(), key=lambda t: t[1]))

What is the minimum expected salary to earn in each state?¶

min_state_salaries

OrderedDict([('NV', 55000),
             ('AZ', 60000),
             ('IN', 67000),
             ('IA', 67500),
             ('ME', 70000),
             ('MI', 70000),
             ('NE', 70000),
             ('AL', 75000),
             ('CO', 80000),
             ('KS', 80000),
             ('OH', 80000),
             ('RI', 80000),
             ('UT', 80000),
             ('SC', 82500),
             ('FL', 83333),
             ('MO', 86000),
             ('TN', 86666),
             ('MN', 90000),
             ('WI', 90000),
             ('NC', 90625),
             ('ON', 92500),
             ('MA', 103076),
             ('MD', 105833),
             ('PA', 108125),
             ('NH', 110000),
             ('VA', 110454),
             ('WA', 111428),
             ('NY', 112750),
             ('TX', 114285),
             ('CT', 115000),
             ('NJ', 120000),
             ('OR', 120000),
             ('IL', 120833),
             ('CA', 123017),
             ('DC', 140000)])

plt.rcParams['figure.figsize'] = (20,10)
min_state_names = list(min_state_salaries.keys())
min_state_values = sorted(list(min_state_salaries.values()))
min_state_y_pos = np.arange(len(min_state_names))

plt.barh(min_state_y_pos, min_state_values, align = 'center',alpha = 0.5)
plt.yticks(min_state_y_pos, min_state_names)
plt.xlabel('Expected Salary')
plt.title('Expected Minimum Earnings Per State')
plt.show()

What is the maximum expected salary to each in each state?¶

max_state_salaries

OrderedDict([('NV', 67500),
             ('AZ', 80000),
             ('RI', 85000),
             ('ME', 90000),
             ('IA', 95000),
             ('MI', 96000),
             ('IN', 96250),
             ('AL', 98750),
             ('KS', 100000),
             ('UT', 100000),
             ('TN', 103333),
             ('CO', 105000),
             ('MN', 110000),
             ('NE', 110000),
             ('FL', 113333),
             ('MO', 116000),
             ('NC', 116875),
             ('OH', 120000),
             ('WI', 120000),
             ('SC', 125000),
             ('ON', 135000),
             ('PA', 137500),
             ('MD', 140000),
             ('NH', 140000),
             ('NJ', 142500),
             ('MA', 145769),
             ('OR', 150000),
             ('TX', 152142),
             ('VA', 157272),
             ('NY', 160875),
             ('DC', 166666),
             ('CA', 171686),
             ('CT', 175000),
             ('WA', 175714),
             ('IL', 197500)])

plt.rcParams['figure.figsize'] = (20,10)
max_state_names = list(max_state_salaries.keys())
max_state_values = sorted(list(max_state_salaries.values()))
max_state_y_pos = np.arange(len(max_state_names))

plt.barh(max_state_y_pos, max_state_values, align = 'center',alpha = 0.5)
plt.yticks(max_state_y_pos, max_state_names)
plt.xlabel('Expected Salary')
plt.title('Expected Maximum Earnings Per State')
plt.show()

What are the expected earnings for the top 10 highest minimum and maximum location salaries?¶

min_location_df = master_data[master_data['Min_Salary'] != 'Unknown'].reset_index(drop = True)
max_location_df = master_data[master_data['Max_Salary'] != 'Unknown'].reset_index(drop = True)

min_location_set = np.unique(min_location_df['Location'])
max_location_set = np.unique(max_location_df['Location'])

min_location_dict = dict(tuple(min_location_df.groupby('Location')))
max_location_dict = dict(tuple(max_location_df.groupby('Location')))

min_location_salaries = {}
max_location_salaries = {}
for state in min_location_set:
    LOCavg_min_salary = int(min_location_dict[state]['Min_Salary'].mean())
    min_location_salaries.update({state : LOCavg_min_salary})

for state in max_location_set:
    LOCavg_max_salary = int(max_location_dict[state]['Max_Salary'].mean())
    max_location_salaries.update({state : LOCavg_max_salary})
    
min_location_salaries = OrderedDict(sorted(min_location_salaries.items(), key=lambda t: t[1]))
max_location_salaries = OrderedDict(sorted(max_location_salaries.items(), key=lambda t: t[1]))

from collections import Counter
min_location_salaries = collections.Counter(min_location_salaries).most_common(10)
max_location_salaries = collections.Counter(max_location_salaries).most_common(10)

min_location_salaries = OrderedDict(sorted(dict(min_location_salaries).items(), key = lambda t:t[1]))
max_location_salaries = OrderedDict(sorted(dict(max_location_salaries).items(), key = lambda t:t[1]))

min_location_salaries

OrderedDict([('Long Beach, CA', 140000),
             ('Ventura County, CA', 140000),
             ('Washington, DC', 140000),
             ('San Mateo, CA', 142500),
             ('Brooklyn, NY', 150000),
             ('Milpitas, CA', 156666),
             ('Sunnyvale, CA', 158333),
             ('Palo Alto, CA', 160000),
             ('Santa Clara, CA', 160000),
             ('Cupertino, CA', 175000)])

max_location_salaries

OrderedDict([('San Jose, CA', 198636),
             ('Annapolis Junction, MD', 200000),
             ('Brooklyn, NY', 200000),
             ('Fairfield, CT', 200000),
             ('Santa Clara, CA', 200000),
             ('Seattle, WA', 205000),
             ('Palo Alto, CA', 206428),
             ('Chicago, IL', 220000),
             ('West Hollywood, CA', 250000),
             ('Cupertino, CA', 325000)])

plt.rcParams['figure.figsize'] = (15,7.5)
min_location_names = list(min_location_salaries.keys())
min_location_values = sorted(list(min_location_salaries.values()))
min_location_y_pos = np.arange(len(min_location_names))

plt.barh(min_location_y_pos, min_location_values, align = 'center',alpha = 0.5)
plt.yticks(min_location_y_pos, min_location_names)
plt.xlabel('Expected Salary')
plt.title('Top 10 Expected Minimum Earnings Per Location')
plt.show()

plt.rcParams['figure.figsize'] = (15,7.5)
max_location_names = list(max_location_salaries.keys())
max_location_values = sorted(list(max_location_salaries.values()))
max_location_y_pos = np.arange(len(max_location_names))

plt.barh(max_location_y_pos, max_location_values, align = 'center',alpha = 0.5)
plt.yticks(max_location_y_pos, max_location_names)
plt.xlabel('Expected Salary')
plt.title('Top 10 Expected Maximum Earnings Per Location')
plt.show()

What are the expected earnings minimum and maximum earnings for each job type (Data Scientist, Data Engineer, Data Analyst, and Business Intelligence)?¶

min_job_df = master_data[master_data['Min_Salary'] != 'Unknown'].reset_index(drop = True)
max_job_df = master_data[master_data['Max_Salary'] != 'Unknown'].reset_index(drop = True)

min_job_set = np.unique(min_job_df['Search'])
max_job_set = np.unique(max_job_df['Search'])

min_job_dict = dict(tuple(min_job_df.groupby('Search')))
max_job_dict = dict(tuple(max_job_df.groupby('Search')))

min_job_salaries = {}
max_job_salaries = {}
for job in min_job_set:
    avg_MINjob_salary = int(min_job_dict[job]['Min_Salary'].mean())
    min_job_salaries.update({job : avg_MINjob_salary})

for job in max_job_set:
    avg_MAXjob_salary = int(max_job_dict[job]['Max_Salary'].mean())
    max_job_salaries.update({job : avg_MAXjob_salary})
    
min_job_salaries = OrderedDict(sorted(min_job_salaries.items(), key=lambda t: t[1]))
max_job_salaries = OrderedDict(sorted(max_job_salaries.items(), key=lambda t: t[1]))

min_job_salaries

OrderedDict([('Data Analyst', 73793),
             ('Data Engineer', 114073),
             ('Business Intelligence', 115757),
             ('Data Scientist', 121800)])

max_job_salaries

OrderedDict([('Data Analyst', 96428),
             ('Business Intelligence', 153125),
             ('Data Engineer', 158706),
             ('Data Scientist', 176800)])

plt.rcParams['figure.figsize'] = (20,10)
min_job_names = list(min_job_salaries.keys())
min_job_values = sorted(list(min_job_salaries.values()))
min_job_y_pos = np.arange(len(min_job_names))

plt.barh(min_job_y_pos, min_job_values, align = 'center',alpha = 0.5)
plt.yticks(min_job_y_pos, min_job_names)
plt.xlabel('Expected Salary')
plt.title('Expected Minimum Earnings Per Job Type')
plt.show()

plt.rcParams['figure.figsize'] = (20,10)
max_job_names = list(max_job_salaries.keys())
max_job_values = sorted(list(max_job_salaries.values()))
max_job_y_pos = np.arange(len(max_job_names))

plt.barh(max_job_y_pos, max_job_values, align = 'center',alpha = 0.5)
plt.yticks(max_job_y_pos, max_job_names)
plt.xlabel('Expected Salary')
plt.title('Expected Maximum Earnings Per Job Type')
plt.show()

exploratory analysis¶

import pandas as pd
import numpy as np
import plotnine as gg
import seaborn as sns
%matplotlib inline
from matplotlib import pyplot as plt
plt.style.use('ggplot')
%matplotlib inline
import geopandas as gpd         
import osmnx as ox              
import shapely.geometry as geom 
df = pd.read_csv('jobs_data.csv')

df.dtypes

Search          object
Title           object
Description     object
Skills          object
Location        object
Cities          object
States          object
Latitude       float64
Longitude      float64
Min_Salary      object
Max_Salary      object
Mean_Salary     object
URL             object
dtype: object

df.Mean_Salary = pd.to_numeric(df.Mean_Salary, errors='coerce')
df.Max_Salary = pd.to_numeric(df.Max_Salary, errors='coerce')
df.Min_Salary= pd.to_numeric(df.Min_Salary, errors='coerce')

df1 = df.dropna(subset = ['Mean_Salary', 'Min_Salary', 'Max_Salary'])

distribution of mean salary¶

# histogram
plt.figure(figsize=(6,6))
sns.distplot(df1['Mean_Salary'].dropna(),color = 'blue')

<matplotlib.axes._subplots.AxesSubplot at 0x203d2c19f60>

Is there a different distribution of location(State, City)/Job Title(data scientist/analyst/engineer/BI) of mean salary?¶

#histogram of price based on different job titles
job_type = df1.Search.unique()

plt.subplots(figsize=(10,8))
for t in np.ndenumerate(job_type):
    kind = df1[df1['Search'] == t[1]]
    sns.distplot(kind.Mean_Salary.dropna(), label=t[1], norm_hist = True)
    
sns.set(rc={'figure.figsize':(11.7,8.27)})
plt.legend(loc=2)
plt.title('Histogram of Mean Salary for Different Job Title')
plt.xlabel('Mean Salary')
plt.show()

#histogram of price based on different states
job_type = df1.States.unique()

plt.subplots(figsize=(15,15))
for t in np.ndenumerate(job_type):
    kind = df1[df1['States'] == t[1]]
    sns.distplot(kind.Mean_Salary.dropna(), label=t[1], norm_hist = True)
    
sns.set(rc={'figure.figsize':(11.7,8.27)})
plt.legend(loc=2)
plt.title('Histogram of Mean Salary for Different Job Title')
plt.xlabel('Mean Salary')
plt.show()

C:\Users\qxzhao\Anaconda3\lib\site-packages\numpy\core\_methods.py:135: RuntimeWarning: Degrees of freedom <= 0 for slice
  keepdims=keepdims)
C:\Users\qxzhao\Anaconda3\lib\site-packages\numpy\core\_methods.py:127: RuntimeWarning: invalid value encountered in double_scalars
  ret = ret.dtype.type(ret / rcount)
C:\Users\qxzhao\Anaconda3\lib\site-packages\statsmodels\nonparametric\bandwidths.py:22: RuntimeWarning: invalid value encountered in minimum
  return np.minimum(np.std(X, axis=0, ddof=1), IQR)
C:\Users\qxzhao\Anaconda3\lib\site-packages\numpy\core\_methods.py:26: RuntimeWarning: invalid value encountered in reduce
  return umr_maximum(a, axis, None, out, keepdims)
C:\Users\qxzhao\Anaconda3\lib\site-packages\statsmodels\nonparametric\kde.py:494: RuntimeWarning: invalid value encountered in true_divide
  binned = fast_linbin(X,a,b,gridsize)/(delta*nobs)
C:\Users\qxzhao\Anaconda3\lib\site-packages\statsmodels\nonparametric\kdetools.py:34: RuntimeWarning: invalid value encountered in double_scalars
  FAC1 = 2*(np.pi*bw/RANGE)**2

df1.groupby(['States','Search']).agg({'Mean_Salary': 'mean'}).head()

from scipy.stats import norm
from sklearn.preprocessing import StandardScaler
from scipy import stats

# Pair plot to explore variables
with sns.plotting_context("notebook",font_scale=1.5):
    g = sns.pairplot(df1, vars = ['Mean_Salary','Min_Salary','Max_Salary','Longitude','Latitude'], hue='Search',
                    palette= sns.hls_palette(4),size=2)
g.set(xticklabels=[]);

# seaborn correlation matrix
str_list = []
for colname, colvalue in df1.iteritems():
    if type(colvalue[1]) == str:
         str_list.append(colname)
            
# Only compare numerical features          
diff_list = df1.columns.difference(str_list) 
df2 = df1[diff_list]
f, ax = plt.subplots(figsize=(12, 8))
plt.title('Pearson Correlation Matrix')

sns.heatmap(df2.astype(float).corr(),linewidths=0.25,vmax=1.0, square=True, cmap="GnBu", linecolor='k', annot=True)
plt.show()

Linear regression model of salary¶

from sklearn import linear_model
from sklearn.metrics import mean_squared_error 
from sklearn import preprocessing

df2 = df1[['Mean_Salary','Latitude','Longitude']]
df2.dtypes

Mean_Salary    float64
Latitude       float64
Longitude      float64
dtype: object

from sklearn.cross_validation import train_test_split
train_data, test_data = train_test_split(df2, train_size = 0.8, random_state = 10)

Simple Linear Regression (One Variable)

def simple_linear(train, test, feature):
    """simple linear regression function specified for the use of our dataset 
    Args:
        train(dataframe): training dataset
        test(dataframe): testing dataset
        feature(Series): features in a dataset
    Returns:
        RMSE(float): Root Mean Squared Error
        regr.intercept (float): regression interception
        regr.coefficient (float): regression coefficient
    """
    regr = linear_model.LinearRegression()
    #Train the model
    regr.fit(train.as_matrix(columns = [feature]), train.as_matrix(columns = ['Mean_Salary']))
    # Calculate the RMSE on test data
    RMSE = mean_squared_error(test.as_matrix(columns = ['Mean_Salary']), 
                              regr.predict(test.as_matrix(columns = [feature])))**0.5 
    return RMSE, regr.intercept_[0], regr.coef_[0][0]

#create an input list, remove the variables we are predicting and non-numerical data
input_list = train_data.columns.values.tolist()
input_list.remove('Mean_Salary') 

simple_linear_result = pd.DataFrame(columns = ['feature', 'RMSE', 'intercept', 'coefficient'])

# Calculate the RMSE of the test data for each feature
for p in input_list:
    RMSE, w1, w0 = simple_linear(train_data, test_data, p)
    simple_linear_result = simple_linear_result.append({'feature':p, 'RMSE':RMSE, 'intercept':w0, 'coefficient': w1}
                                                        ,ignore_index=True)
simple_linear_result

#plot scatterplot 
plt.figure(figsize=(8, 5))
ax = sns.regplot(x = df2.Latitude, y = df2.Mean_Salary)
plt.title('Latitude vs.Mean_Salary')
plt.xlabel('Latitude')
plt.ylabel('Mean Salary')

plt.tight_layout()

#plot scatterplot 
plt.figure(figsize=(8, 5))
ax = sns.regplot(x = df2.Longitude, y = df2.Mean_Salary)
plt.title('Longitude vs.Mean_Salary')
plt.xlabel('Longitude')
plt.ylabel('Mean Salary')

plt.tight_layout()

Multi Linear Regression (Two Variables)

# further split the train_data to include a validation set
#train_data2 = 60%, validation_data = 20%, test_data = 20%
train_data_2, validation_data = train_test_split(train_data, train_size = 0.75, random_state = 50)

def RMSE(train, validation, features, new_input):
    """RMSE specified for the return RMSE of features
    Args:
        train(dataframe): training dataset
        validation(dataframe): validation dataset
        feature(Series): features in a dataset
        new_input (feature): reminding feature in the dataset
    Returns:
        RMSE(float): Root Mean Squared Error of train data
        RMSE(float): Root Mean Squared Error of validation data
    """
    features_list = list(features)
    features_list.append(new_input)
    regr = linear_model.LinearRegression() 
    regr.fit(train.as_matrix(columns = features_list), train.as_matrix(columns = ['Mean_Salary'])) # Train the model
    RMSE_train = mean_squared_error(train.as_matrix(columns = ['Mean_Salary']), 
                              regr.predict(train.as_matrix(columns = features_list)))**0.5 # Calculate the RMSE on train data
    RMSE_validation = mean_squared_error(validation.as_matrix(columns = ['Mean_Salary']), 
                              regr.predict(validation.as_matrix(columns = features_list)))**0.5 # Calculate the RMSE on train data
    return RMSE_train, RMSE_validation

input_list = train_data_2.columns.values.tolist()
input_list.remove('Mean_Salary')

greedy_algorithm = pd.DataFrame(columns = ['feature', 'train_error', 'validation_error'])  
i = 0
temp_list = []


# Go through all the features in the dataframe
while len(input_list)>0:
    
    # temporarily store the best feature at each iteration
    temp = pd.DataFrame(columns = ['feature', 'train_error', 'validation_error'])
    
    # to test all the remaining features
    for p in input_list:
        RMSE_train, RMSE_validation = RMSE(train_data_2, validation_data, temp_list, p)
        temp = temp.append({'feature':p, 'train_error':RMSE_train, 'validation_error':RMSE_validation}, ignore_index=True)
    
    # select the best feature with smallest train error
    temp = temp.sort_values('train_error')
    best = temp.iloc[0,0]
    temp_list.append(best)
    greedy_algorithm = greedy_algorithm.append({'feature': best, 
                                                  'train_error': temp.iloc[0,1], 'validation_error': temp.iloc[0,2]}, 
                                                 ignore_index=True)
    #remove the best feature before next iteration
    input_list.remove(best)
    i += 1
greedy_algorithm

def multiple_regression_model(train, test, input_features):
    """modifided version of simple linear regression function to take mutliple features
    Args:
        train(dataframe): training dataset
        test(dataframe): testing dataset
        input_feature(Series): features in a dataset
    Returns:
        RMSE(float): Root Mean Squared Error
        regr.intercept (float): regression interception
        regr.coefficient (float): regression coefficient
    """
    regr = linear_model.LinearRegression() # Create a linear regression object
    regr.fit(train.as_matrix(columns = input_features), train.as_matrix(columns = ['Mean_Salary'])) # Train the model
    RMSE = mean_squared_error(test.as_matrix(columns = ['Mean_Salary']), 
                              regr.predict(test.as_matrix(columns = input_features)))**0.5 # Calculate the RMSE on test data
    return RMSE, regr.intercept_[0], regr.coef_

greedy_algo_features_list = greedy_algorithm['feature'].tolist()[:4]
test_error, _, _ = multiple_regression_model(train_data_2, test_data, greedy_algo_features_list)
print ('test error (RMSE) is: %s' %test_error)

test error (RMSE) is: 38388.02845162767

Mean_Salary = -14648.4802 + Longitude x -575.3558 + Latitude x 2448.8531

multiple_regression_model(train_data_2, test_data, greedy_algo_features_list)

(38388.02845162767,
 -14648.48016281589,
 array([[-575.35579643, 2448.85309993]]))

	Search	Frequency	Percent of Total
0	Data Engineer	349	0.6724
1	Data Scientist	72	0.1387
2	Data Analyst	49	0.0944
3	Business Intelligence	49	0.0944

	States	Frequency	Percent of Total
0	CA	255	0.4913
1	NY	50	0.0963
2	MA	27	0.0520
3	VA	20	0.0385
4	TX	15	0.0289
5	WA	14	0.0270
6	PA	12	0.0231
7	MD	11	0.0212
8	NC	11	0.0212
9	IL	9	0.0173
10	NJ	9	0.0173
11	GA	8	0.0154
12	MO	7	0.0135
13	NV	6	0.0116
14	MI	6	0.0116
15	DC	6	0.0116
16	TN	5	0.0096
17	IN	5	0.0096
18	CO	5	0.0096
19	SC	5	0.0096
20	AL	4	0.0077
21	FL	4	0.0077
22	MN	3	0.0058
23	IA	2	0.0039
24	RI	2	0.0039
25	NH	2	0.0039
26	NE	2	0.0039
27	OH	2	0.0039
28	ON	2	0.0039
29	CT	2	0.0039
30	OR	1	0.0019
31	AB	1	0.0019
32	ME	1	0.0019
33	WI	1	0.0019
34	KS	1	0.0019
35	KY	1	0.0019
36	UT	1	0.0019
37	AZ	1	0.0019

	States	Search	Frequency	Percent of Total
0	AB	Data Scientist	1	1.0000
1	AL	Data Engineer	3	0.7500
2	AL	Business Intelligence	1	0.2500
3	AZ	Data Analyst	1	1.0000
4	CA	Data Engineer	178	0.6980
5	CA	Data Scientist	35	0.1373
6	CA	Business Intelligence	23	0.0902
7	CA	Data Analyst	19	0.0745
8	CO	Data Engineer	5	1.0000
9	CT	Business Intelligence	1	0.5000
10	CT	Data Engineer	1	0.5000
11	DC	Data Engineer	5	0.8333
12	DC	Data Analyst	1	0.1667
13	FL	Data Engineer	4	1.0000
14	GA	Data Engineer	4	0.5000
15	GA	Data Analyst	3	0.3750
16	GA	Business Intelligence	1	0.1250
17	IA	Data Engineer	1	0.5000
18	IA	Data Scientist	1	0.5000
19	IL	Data Engineer	7	0.7778
20	IL	Business Intelligence	1	0.1111
21	IL	Data Scientist	1	0.1111
22	IN	Business Intelligence	3	0.6000
23	IN	Data Analyst	1	0.2000
24	IN	Data Engineer	1	0.2000
25	KS	Data Scientist	1	1.0000
26	KY	Data Engineer	1	1.0000
27	MA	Data Engineer	20	0.7407
28	MA	Data Scientist	4	0.1481
29	MA	Data Analyst	3	0.1111
30	MD	Data Engineer	10	0.9091
31	MD	Data Scientist	1	0.0909
32	ME	Business Intelligence	1	1.0000
33	MI	Data Engineer	6	1.0000
34	MN	Data Engineer	3	1.0000
35	MO	Data Engineer	4	0.5714
36	MO	Business Intelligence	2	0.2857
37	MO	Data Analyst	1	0.1429
38	NC	Data Engineer	9	0.8182
39	NC	Data Analyst	1	0.0909
40	NC	Data Scientist	1	0.0909
41	NE	Data Engineer	2	1.0000
42	NH	Data Engineer	2	1.0000
43	NJ	Data Analyst	4	0.4444
44	NJ	Data Engineer	3	0.3333
45	NJ	Data Scientist	2	0.2222
46	NV	Data Analyst	4	0.6667
47	NV	Data Engineer	2	0.3333
48	NY	Data Engineer	23	0.4600
49	NY	Data Scientist	14	0.2800
50	NY	Business Intelligence	9	0.1800
51	NY	Data Analyst	4	0.0800
52	OH	Data Engineer	2	1.0000
53	ON	Data Engineer	2	1.0000
54	OR	Data Engineer	1	1.0000
55	PA	Data Engineer	7	0.5833
56	PA	Data Scientist	3	0.2500
57	PA	Data Analyst	2	0.1667
58	RI	Data Analyst	1	0.5000
59	RI	Data Engineer	1	0.5000
60	SC	Data Engineer	3	0.6000
61	SC	Data Analyst	2	0.4000
62	TN	Data Scientist	3	0.6000
63	TN	Data Engineer	2	0.4000
64	TX	Data Engineer	9	0.6000
65	TX	Business Intelligence	5	0.3333
66	TX	Data Scientist	1	0.0667
67	UT	Data Engineer	1	1.0000
68	VA	Data Engineer	17	0.8500
69	VA	Data Scientist	2	0.1000
70	VA	Data Analyst	1	0.0500
71	WA	Data Engineer	10	0.7143
72	WA	Data Scientist	2	0.1429
73	WA	Business Intelligence	1	0.0714
74	WA	Data Analyst	1	0.0714
75	WI	Business Intelligence	1	1.0000

	States	Cities	Frequency
0	CA	San Francisco	49
1	NY	New York City	36
2	CA	San Mateo	22
3	CA	San Jose	22
4	CA	Irvine	20
5	CA	Los Angeles	18
6	CA	Palo Alto	11
7	WA	Seattle	10
8	CA	Redwood City	8
9	MA	Cambridge	8
10	IL	Chicago	8
11	GA	Atlanta	7
12	VA	Reston	7
13	CA	Santa Monica	7
14	CA	Sunnyvale	7
15	VA	Arlington	7
16	MA	Burlington	6
17	NV	Las Vegas	6
18	TX	Austin	6
19	CA	San Diego	6
20	DC	Washington	6
21	CA	Venice	6
22	NY	White Plains	5
23	CA	San Ramon	5
24	CA	Mountain View	5
25	NC	Raleigh	5
26	PA	Pittsburgh	5
27	MI	Detroit	4
28	PA	Philadelphia	4
29	MO	Saint Louis	4
...	...	...	...
132	AL	Huntsville	1
133	AL	Daphne	1
134	CA	Van Nuys	1
135	CA	Ventura County	1
136	CA	Westwood	1
137	CO	Boulder	1
138	NC	Durham	1
139	NC	Charlotte	1
140	MO	Fenton	1
141	MN	Plymouth	1
142	MI	Wixom	1
143	MI	Troy	1
144	ME	Portland	1
145	MD	Rockville	1
146	MD	Camp Springs	1
147	MD	Bowie	1
148	MD	Baltimore	1
149	MD	Anne Arundel County	1
150	MD	Annapolis Junction	1
151	KY	Lexington	1
152	KS	Leawood	1
153	IL	Peoria	1
154	GA	Dunwoody	1
155	FL	Tampa	1
156	FL	Boca Raton	1
157	CT	Torrington	1
158	CT	Fairfield	1
159	CO	Loveland	1
160	CO	Denver	1
161	WI	Berlin	1

	States	Cities	Frequency	Percent of Total
0	AB	Calgary	1	1.0000
1	AL	Albertville	2	0.5000
2	AL	Huntsville	1	0.2500
3	AL	Daphne	1	0.2500
4	AZ	Scottsdale	1	1.0000
5	CA	Palm Springs	1	0.0039
6	CA	Pasadena	1	0.0039
7	CA	Redondo Beach	1	0.0039
8	CA	Roseville	1	0.0039
9	CA	Santa Ana	1	0.0039
10	CA	Torrance	1	0.0039
11	CA	North Hollywood	1	0.0039
12	CA	Marina Del Rey	1	0.0039
13	CA	Pleasanton	2	0.0078
14	CA	Canoga Park	2	0.0078
15	CA	Santa Barbara	1	0.0039
16	CA	Newport Beach	1	0.0039
17	CA	Fremont	1	0.0039
18	CA	Long Beach	1	0.0039
19	CA	El Segundo	2	0.0078
20	CA	Foster City	1	0.0039
21	CA	Cupertino	1	0.0039
22	CA	Concord	1	0.0039
23	CA	Carlsbad	1	0.0039
24	CA	Campbell	1	0.0039
25	CA	Beverly Hills	1	0.0039
26	CA	Aliso Viejo	1	0.0039
27	CA	Van Nuys	1	0.0039
28	CA	Ventura County	1	0.0039
29	CA	Westwood	1	0.0039
30	CA	Napa	1	0.0039
31	CA	Gardena	2	0.0078
32	CA	San Carlos	1	0.0039
33	CA	Menlo Park	2	0.0078
34	CA	San Francisco	49	0.1922
35	CA	San Mateo	22	0.0863
36	CA	Laguna Niguel	2	0.0078
37	CA	Irvine	20	0.0784
38	CA	Los Angeles	18	0.0706
39	CA	Palo Alto	11	0.0431
40	CA	Redwood City	8	0.0314
41	CA	Santa Monica	7	0.0275
42	CA	Sunnyvale	7	0.0275
43	CA	San Diego	6	0.0235
44	CA	Venice	6	0.0235
45	CA	San Ramon	5	0.0196
46	CA	Mountain View	5	0.0196
47	CA	San Jose	22	0.0863
48	CA	Milpitas	3	0.0118
49	CA	Northridge	2	0.0078
50	CA	Norwalk	2	0.0078
51	CA	Orange	2	0.0078
52	CA	Ventura	3	0.0118
53	CA	Glendale	3	0.0118
54	CA	Santa Clara	2	0.0078
55	CA	West Hollywood	3	0.0118
56	CA	Woodland Hills	3	0.0118
57	CA	Culver City	3	0.0118
58	CA	Costa Mesa	3	0.0118
59	CA	Berkeley	4	0.0157
60	CO	Denver	1	0.2000
61	CO	Colorado Springs	2	0.4000
62	CO	Boulder	1	0.2000
63	CO	Loveland	1	0.2000
64	CT	Fairfield	1	0.5000
65	CT	Torrington	1	0.5000
66	DC	Washington	6	1.0000
67	FL	Orlando	2	0.5000
68	FL	Tampa	1	0.2500
69	FL	Boca Raton	1	0.2500
70	GA	Atlanta	7	0.8750
71	GA	Dunwoody	1	0.1250
72	IA	Grinnell	2	1.0000
73	IL	Chicago	8	0.8889
74	IL	Peoria	1	0.1111
75	IN	South Bend	3	0.6000
76	IN	Indianapolis	2	0.4000
77	KS	Leawood	1	1.0000
78	KY	Lexington	1	1.0000
79	MA	Bedford	2	0.0741
80	MA	Wellesley	2	0.0741
81	MA	Acton	3	0.1111
82	MA	Burlington	6	0.2222
83	MA	Newton	3	0.1111
84	MA	Cambridge	8	0.2963
85	MA	Boston	3	0.1111
86	MD	Bethesda	3	0.2727
87	MD	Fort Meade	2	0.1818
88	MD	Rockville	1	0.0909
89	MD	Camp Springs	1	0.0909
90	MD	Bowie	1	0.0909
91	MD	Baltimore	1	0.0909
92	MD	Anne Arundel County	1	0.0909
93	MD	Annapolis Junction	1	0.0909
94	ME	Portland	1	1.0000
95	MI	Detroit	4	0.6667
96	MI	Wixom	1	0.1667
97	MI	Troy	1	0.1667
98	MN	Saint Paul	2	0.6667
99	MN	Plymouth	1	0.3333
100	MO	Fenton	1	0.1429
101	MO	Kansas City	2	0.2857
102	MO	Saint Louis	4	0.5714
103	NC	Charlotte	1	0.0909
104	NC	Durham	1	0.0909
105	NC	Huntersville	1	0.0909
106	NC	Raleigh	5	0.4545
107	NC	Youngsville	1	0.0909
108	NC	Randleman	1	0.0909
109	NC	Pineville	1	0.0909
110	NE	Omaha	2	1.0000
111	NH	Manchester	2	1.0000
112	NJ	Jersey City	4	0.4444
113	NJ	Princeton	3	0.3333
114	NJ	Trenton	1	0.1111
115	NJ	Hazlet	1	0.1111
116	NV	Las Vegas	6	1.0000
117	NY	Melville	1	0.0200
118	NY	Stony Brook	1	0.0200
119	NY	Port Washington	1	0.0200
120	NY	Rochester	1	0.0200
121	NY	Manhattan	1	0.0200
122	NY	East Setauket	1	0.0200
123	NY	New York City	36	0.7200
124	NY	Brooklyn	1	0.0200
125	NY	Elmira	2	0.0400
126	NY	White Plains	5	0.1000
127	OH	Toledo	1	0.5000
128	OH	Solon	1	0.5000
129	ON	Waterloo	1	0.5000
130	ON	Kitchener	1	0.5000
131	OR	Portland	1	1.0000
132	PA	Pittsburgh	5	0.4167
133	PA	Philadelphia	4	0.3333
134	PA	Malvern	2	0.1667
135	PA	Plymouth Meeting	1	0.0833
136	RI	Providence	2	1.0000
137	SC	Rock Hill	1	0.2000
138	SC	Charleston	3	0.6000
139	SC	Mount Pleasant	1	0.2000
140	TN	Johnson City	1	0.2000
141	TN	Nashville	2	0.4000
142	TN	Knoxville	2	0.4000
143	TX	Austin	6	0.4000
144	TX	Plano	2	0.1333
145	TX	Houston	2	0.1333
146	TX	Dallas	2	0.1333
147	TX	Arlington	1	0.0667
148	TX	Katy	1	0.0667
149	TX	Missouri City	1	0.0667
150	UT	Lindon	1	1.0000
151	VA	Reston	7	0.3500
152	VA	Arlington	7	0.3500
153	VA	Alexandria	2	0.1000
154	VA	Mclean	2	0.1000
155	VA	Timberlake	1	0.0500
156	VA	Chantilly	1	0.0500
157	WA	Bellevue	1	0.0714
158	WA	Yakima	1	0.0714
159	WA	Seattle	10	0.7143
160	WA	Tacoma	2	0.1429
161	WI	Berlin	1	1.0000

	Location	Search	Frequency
0	New York City, NY	Data Scientist	13
1	San Francisco, CA	Data Scientist	12
2	Los Angeles, CA	Data Scientist	4
3	Sunnyvale, CA	Data Scientist	3
4	San Mateo, CA	Data Scientist	3
5	Berkeley, CA	Data Scientist	2
6	Cambridge, MA	Data Scientist	2
7	Nashville, TN	Data Scientist	2
8	San Jose, CA	Data Scientist	2
9	West Hollywood, CA	Data Scientist	2
10	Trenton, NJ	Data Scientist	1
11	Seattle, WA	Data Scientist	1
12	Santa Monica, CA	Data Scientist	1
13	Redwood City, CA	Data Scientist	1
14	Plymouth Meeting, PA	Data Scientist	1
15	Pittsburgh, PA	Data Scientist	1
16	Pineville, NC	Data Scientist	1
17	Philadelphia, PA	Data Scientist	1
18	Newton, MA	Data Scientist	1
19	Alexandria, VA	Data Scientist	1
20	Napa, CA	Data Scientist	1
21	Chicago, IL	Data Scientist	1
22	Bedford, MA	Data Scientist	1
23	Bellevue, WA	Data Scientist	1
24	Bethesda, MD	Data Scientist	1
25	Calgary, AB	Data Scientist	1
26	Campbell, CA	Data Scientist	1
27	Chantilly, VA	Data Scientist	1
28	Costa Mesa, CA	Data Scientist	1
29	Austin, TX	Data Scientist	1
30	Elmira, NY	Data Scientist	1
31	Foster City, CA	Data Scientist	1
32	Grinnell, IA	Data Scientist	1
33	Hazlet, NJ	Data Scientist	1
34	Knoxville, TN	Data Scientist	1
35	Leawood, KS	Data Scientist	1
36	Woodland Hills, CA	Data Scientist	1

	Location	Search	Frequency
0	New York City, NY	Data Analyst	4
1	Las Vegas, NV	Data Analyst	4
2	Atlanta, GA	Data Analyst	3
3	Ventura, CA	Data Analyst	3
4	Princeton, NJ	Data Analyst	3
5	Laguna Niguel, CA	Data Analyst	2
6	Pleasanton, CA	Data Analyst	2
7	Woodland Hills, CA	Data Analyst	2
8	Orange, CA	Data Analyst	2
9	Cambridge, MA	Data Analyst	2
10	San Ramon, CA	Data Analyst	2
11	Venice, CA	Data Analyst	1
12	Scottsdale, AZ	Data Analyst	1
13	Washington, DC	Data Analyst	1
14	San Francisco, CA	Data Analyst	1
15	Rock Hill, SC	Data Analyst	1
16	Randleman, NC	Data Analyst	1
17	Providence, RI	Data Analyst	1
18	Palm Springs, CA	Data Analyst	1
19	Pittsburgh, PA	Data Analyst	1
20	Philadelphia, PA	Data Analyst	1
21	Newton, MA	Data Analyst	1
22	Mount Pleasant, SC	Data Analyst	1
23	Mclean, VA	Data Analyst	1
24	Los Angeles, CA	Data Analyst	1
25	Kansas City, MO	Data Analyst	1
26	Jersey City, NJ	Data Analyst	1
27	Indianapolis, IN	Data Analyst	1
28	El Segundo, CA	Data Analyst	1
29	Canoga Park, CA	Data Analyst	1
30	Yakima, WA	Data Analyst	1

	Location	Search	Frequency
0	Irvine, CA	Business Intelligence	7
1	White Plains, NY	Business Intelligence	5
2	South Bend, IN	Business Intelligence	3
3	San Mateo, CA	Business Intelligence	3
4	Palo Alto, CA	Business Intelligence	3
5	San Francisco, CA	Business Intelligence	2
6	Milpitas, CA	Business Intelligence	2
7	San Diego, CA	Business Intelligence	2
8	New York City, NY	Business Intelligence	2
9	Austin, TX	Business Intelligence	2
10	Redwood City, CA	Business Intelligence	1
11	Torrington, CT	Business Intelligence	1
12	Seattle, WA	Business Intelligence	1
13	Santa Monica, CA	Business Intelligence	1
14	Santa Ana, CA	Business Intelligence	1
15	Saint Louis, MO	Business Intelligence	1
16	Atlanta, GA	Business Intelligence	1
17	Portland, ME	Business Intelligence	1
18	Port Washington, NY	Business Intelligence	1
19	Missouri City, TX	Business Intelligence	1
20	Katy, TX	Business Intelligence	1
21	Kansas City, MO	Business Intelligence	1
22	Houston, TX	Business Intelligence	1
23	Daphne, AL	Business Intelligence	1
24	Costa Mesa, CA	Business Intelligence	1
25	Brooklyn, NY	Business Intelligence	1
26	Berlin, WI	Business Intelligence	1
27	Peoria, IL	Business Intelligence	1

	word	count
0	python	172
1	java	138
2	sql	130
3	machine-learning	100
4	c	88
5	big-data	77
6	hadoop	64
7	aws	59
8	linux	47
9	javascript	45
10	spark	44
11	scala	40
12	etl	36
13	intelligence	33
14	cisco	32
15	network	32
16	cloud	32
17	r	32
18	web	30
19	kafka	30

	word	count
0	learning	66
1	machine	58
2	python	51
3	r	22
4	sql	19
5	mining	13
6	hadoop	13
7	nlp	11
8	big	10
9	java	8
10	spark	8
11	ai	7
12	predictive	7
13	statistical	7
14	product	7
15	risk	6
16	neural	6
17	bayesian	6
18	code	6
19	network	6

	feature	RMSE	intercept	coefficient
0	Latitude	41543.143308	544.935841	112339.426957
1	Longitude	38847.625814	-483.167004	84053.526742

	feature	train_error	validation_error
0	Longitude	38440.581350	31584.593133
1	Latitude	37562.730846	32612.850781

	word	count
0	sql	30
1	excel	18
2	tableau	10
3	financial	10
4	analyst	8
5	python	8
6	automation	8
7	marketing	5
8	e-commerce	5
9	create	5
10	project	5
11	query	4
12	process	4
13	automate	4
14	team	4
15	write	4
16	test	4
17	end	4
18	qa	4
19	ms	4

	word	count
0	intelligence	27
1	sql	19
2	bi	15
3	oracle	12
4	etl	11
5	java	9
6	product	8
7	sales	8
8	enterprise	7
9	one	7
10	net	7
11	marketing	6
12	tableau	6
13	power	6
14	saas	6
15	process	5
16	c	5
17	microsoft	5
18	publisher	5
19	ssrs	4